^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * This file contains the 64-bit "server" PowerPC variant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * of the low level exception handling including exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * vectors, exception return, part of the slb and stab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * handling and other fixed offset specific things.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * This file is meant to be #included from head_64.S due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * position dependent assembly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Most of this originates from head_64.S and thus has the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * copyright history.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/hw_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/exception-64s.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/cpuidle.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/head-64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/feature-fixups.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/kup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /* PACA save area offsets (exgen, exmc, etc) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define EX_R9 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define EX_R10 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define EX_R11 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define EX_R12 24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define EX_R13 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define EX_DAR 40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define EX_DSISR 48
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define EX_CCR 52
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define EX_CFAR 56
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define EX_PPR 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define EX_CTR 72
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) .if EX_SIZE != 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) .error "EX_SIZE is wrong"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * Following are fixed section helper macros.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * EXC_REAL_BEGIN/END - real, unrelocated exception vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * EXC_VIRT_BEGIN/END - virt (AIL), unrelocated exception vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * TRAMP_REAL_BEGIN - real, unrelocated helpers (virt may call these)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * TRAMP_VIRT_BEGIN - virt, unreloc helpers (in practice, real can use)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * EXC_COMMON - After switching to virtual, relocated mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define EXC_REAL_BEGIN(name, start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define EXC_REAL_END(name, start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define EXC_VIRT_BEGIN(name, start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define EXC_VIRT_END(name, start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define EXC_COMMON_BEGIN(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) USE_TEXT_SECTION(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) .balign IFETCH_ALIGN_BYTES; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) .global name; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) _ASM_NOKPROBE_SYMBOL(name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) DEFINE_FIXED_SYMBOL(name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) name:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define TRAMP_REAL_BEGIN(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define TRAMP_VIRT_BEGIN(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define EXC_REAL_NONE(start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define EXC_VIRT_NONE(start, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * We're short on space and time in the exception prolog, so we can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * use the normal LOAD_REG_IMMEDIATE macro to load the address of label.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * Instead we get the base of the kernel from paca->kernelbase and or in the low
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * part of label. This requires that the label be within 64KB of kernelbase, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * that kernelbase be 64K aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define LOAD_HANDLER(reg, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) ld reg,PACAKBASE(r13); /* get high part of &label */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define __LOAD_HANDLER(reg, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ld reg,PACAKBASE(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) ori reg,reg,(ABS_ADDR(label))@l
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * Branches from unrelocated code (e.g., interrupts) to labels outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * head-y require >64K offsets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define __LOAD_FAR_HANDLER(reg, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ld reg,PACAKBASE(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) ori reg,reg,(ABS_ADDR(label))@l; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) addis reg,reg,(ABS_ADDR(label))@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Branch to label using its 0xC000 address. This results in instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * on using mtmsr rather than rfid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * load KBASE for a slight optimisation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define BRANCH_TO_C000(reg, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) __LOAD_FAR_HANDLER(reg, label); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) mtctr reg; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * Interrupt code generation macros
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define IVEC .L_IVEC_\name\() /* Interrupt vector address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define IHSRR .L_IHSRR_\name\() /* Sets SRR or HSRR registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define IHSRR_IF_HVMODE .L_IHSRR_IF_HVMODE_\name\() /* HSRR if HV else SRR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define IAREA .L_IAREA_\name\() /* PACA save area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define IVIRT .L_IVIRT_\name\() /* Has virt mode entry point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #define IISIDE .L_IISIDE_\name\() /* Uses SRR0/1 not DAR/DSISR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define IDAR .L_IDAR_\name\() /* Uses DAR (or SRR0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define IDSISR .L_IDSISR_\name\() /* Uses DSISR (or SRR1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define ISET_RI .L_ISET_RI_\name\() /* Run common code w/ MSR[RI]=1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define __IKVM_REAL(name) .L_IKVM_REAL_ ## name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define __ISTACK(name) .L_ISTACK_ ## name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define INT_DEFINE_BEGIN(n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .macro int_define_ ## n name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define INT_DEFINE_END(n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .endm ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) int_define_ ## n n ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) do_define_int n
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .macro do_define_int name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .ifndef IVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .error "IVEC not defined"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .ifndef IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) IHSRR=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .ifndef IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) IHSRR_IF_HVMODE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .ifndef IAREA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) IAREA=PACA_EXGEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .ifndef IVIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) IVIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) .ifndef IISIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) IISIDE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) .ifndef IDAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) IDAR=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) .ifndef IDSISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) IDSISR=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) .ifndef ISET_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) ISET_RI=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) .ifndef IBRANCH_TO_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) IBRANCH_TO_COMMON=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) .ifndef IREALMODE_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) IREALMODE_COMMON=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) .if ! IBRANCH_TO_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) .error "IREALMODE_COMMON=1 but IBRANCH_TO_COMMON=0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) .ifndef IMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) IMASK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) .ifndef IKVM_SKIP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) IKVM_SKIP=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) .ifndef IKVM_REAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) IKVM_REAL=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) .ifndef IKVM_VIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) IKVM_VIRT=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) .ifndef ISTACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) ISTACK=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) .ifndef IRECONCILE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) IRECONCILE=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) .ifndef IKUAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) IKUAP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * All interrupts which set HSRR registers, as well as SRESET and MCE and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * so they all generally need to test whether they were taken in guest context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * Note: SRESET and MCE may also be sent to the guest by the hypervisor, and be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * taken with MSR[HV]=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * Interrupts which set SRR registers (with the above exceptions) do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * elevate to MSR[HV]=1 mode, though most can be taken when running with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * MSR[HV]=1 (e.g., bare metal kernel and userspace). So these interrupts do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * not need to test whether a guest is running because they get delivered to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * the guest directly, including nested HV KVM guests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * The exception is PR KVM, where the guest runs with MSR[PR]=1 and the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * runs with MSR[HV]=0, so the host takes all interrupts on behalf of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * guest. PR KVM runs with LPCR[AIL]=0 which causes interrupts to always be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * delivered to the real-mode entry point, therefore such interrupts only test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * KVM in their real mode handlers, and only when PR KVM is possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * Interrupts that are taken in MSR[HV]=0 and escalate to MSR[HV]=1 are always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * delivered in real-mode when the MMU is in hash mode because the MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * registers are not set appropriately to translate host addresses. In nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * radix mode these can be delivered in virt-mode as the host translations are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * used implicitly (see: effective LPID, effective PID).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * If an interrupt is taken while a guest is running, it is immediately routed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * to kvmppc_interrupt_hv, which handles the PR guest case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #define kvmppc_interrupt kvmppc_interrupt_hv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #define kvmppc_interrupt kvmppc_interrupt_pr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) .macro KVMTEST name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) lbz r10,HSTATE_IN_GUEST(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) cmpwi r10,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) bne \name\()_kvm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) .macro GEN_KVM name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) .balign IFETCH_ALIGN_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) \name\()_kvm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) .if IKVM_SKIP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) cmpwi r10,KVM_GUEST_MODE_SKIP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) beq 89f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) ld r10,IAREA+EX_CFAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) std r10,HSTATE_CFAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) ld r10,IAREA+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) ld r10,IAREA+EX_PPR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) std r10,HSTATE_PPR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) ld r11,IAREA+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) ld r12,IAREA+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) std r12,HSTATE_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) sldi r12,r9,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) ld r9,IAREA+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ld r10,IAREA+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* HSRR variants have the 0x2 bit added to their trap number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) .if IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) ori r12,r12,(IVEC + 0x2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ori r12,r12,(IVEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) .elseif IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) ori r12,r12,(IVEC+ 0x2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) ori r12,r12,(IVEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) b kvmppc_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) .if IKVM_SKIP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 89: mtocrf 0x80,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) ld r10,IAREA+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) ld r9,IAREA+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) ld r10,IAREA+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) ld r11,IAREA+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) ld r12,IAREA+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) .if IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) b kvmppc_skip_Hinterrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) b kvmppc_skip_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) .elseif IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) b kvmppc_skip_Hinterrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) b kvmppc_skip_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) .macro KVMTEST name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) .macro GEN_KVM name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * This is the BOOK3S interrupt entry code macro.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * This can result in one of several things happening:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * - Branch to the _common handler, relocated, in virtual mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * These are normal interrupts (synchronous and asynchronous) handled by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * - Branch to KVM, relocated but real mode interrupts remain in real mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * / intended for host or guest kernel, but KVM must always be involved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * because the machine state is set for guest execution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * - Branch to the masked handler, unrelocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * These occur when maskable asynchronous interrupts are taken with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * irq_soft_mask set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * - Branch to an "early" handler in real mode but relocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * This is done if early=1. MCE and HMI use these to handle errors in real
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * - Fall through and continue executing in real, unrelocated mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * This is done if early=2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) .macro GEN_BRANCH_TO_COMMON name, virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) .if IREALMODE_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) LOAD_HANDLER(r10, \name\()_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) .if \virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) #ifndef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) b \name\()_common_virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) LOAD_HANDLER(r10, \name\()_common_virt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) LOAD_HANDLER(r10, \name\()_common_real)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) .macro GEN_INT_ENTRY name, virt, ool=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) SET_SCRATCH0(r13) /* save r13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) std r9,IAREA+EX_R9(r13) /* save r9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) mfspr r9,SPRN_PPR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) HMT_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) std r10,IAREA+EX_R10(r13) /* save r10 - r12 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) mfspr r10,SPRN_CFAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) .if \ool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) .if !\virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) b tramp_real_\name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) .pushsection .text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) TRAMP_REAL_BEGIN(tramp_real_\name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) b tramp_virt_\name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) .pushsection .text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) TRAMP_VIRT_BEGIN(tramp_virt_\name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) std r9,IAREA+EX_PPR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) std r10,IAREA+EX_CFAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) INTERRUPT_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) mfctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) std r10,IAREA+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) mfcr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) std r11,IAREA+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) std r12,IAREA+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * because a d-side MCE will clobber those registers so is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * not recoverable if they are live.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) GET_SCRATCH0(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) std r10,IAREA+EX_R13(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) .if IDAR && !IISIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) .if IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) mfspr r10,SPRN_HDAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) mfspr r10,SPRN_DAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) std r10,IAREA+EX_DAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) .if IDSISR && !IISIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) .if IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) mfspr r10,SPRN_HDSISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) mfspr r10,SPRN_DSISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) stw r10,IAREA+EX_DSISR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) .if IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) mfspr r11,SPRN_HSRR0 /* save HSRR0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) mfspr r12,SPRN_HSRR1 /* and HSRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) mfspr r11,SPRN_SRR0 /* save SRR0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) mfspr r12,SPRN_SRR1 /* and SRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) .elseif IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) mfspr r11,SPRN_HSRR0 /* save HSRR0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) mfspr r12,SPRN_HSRR1 /* and HSRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) mfspr r11,SPRN_SRR0 /* save SRR0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) mfspr r12,SPRN_SRR1 /* and SRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) .if IBRANCH_TO_COMMON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) GEN_BRANCH_TO_COMMON \name \virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) .if \ool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * __GEN_COMMON_ENTRY is required to receive the branch from interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * entry, except in the case of the real-mode handlers which require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * __GEN_REALMODE_COMMON_ENTRY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * This switches to virtual mode and sets MSR[RI].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) .macro __GEN_COMMON_ENTRY name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) DEFINE_FIXED_SYMBOL(\name\()_common_real)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) \name\()_common_real:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) .if IKVM_REAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) KVMTEST \name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) ld r10,PACAKMSR(r13) /* get MSR value for kernel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* MSR[RI] is clear iff using SRR regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) .if IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) xori r10,r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) .elseif ! IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) xori r10,r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) mtmsrd r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) .if IVIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) .if IKVM_VIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) b 1f /* skip the virt test coming from real */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) .balign IFETCH_ALIGN_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) DEFINE_FIXED_SYMBOL(\name\()_common_virt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) \name\()_common_virt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) .if IKVM_VIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) KVMTEST \name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) .endif /* IVIRT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * Don't switch to virt mode. Used for early MCE and HMI handlers that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * want to run in real mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) .macro __GEN_REALMODE_COMMON_ENTRY name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) DEFINE_FIXED_SYMBOL(\name\()_common_real)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) \name\()_common_real:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) .if IKVM_REAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) KVMTEST \name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) .macro __GEN_COMMON_BODY name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) .if IMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) .if ! ISTACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) .error "No support for masked interrupt to use custom stack"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /* If coming from user, skip soft-mask tests. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) andi. r10,r12,MSR_PR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) bne 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Kernel code running below __end_interrupts is implicitly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * soft-masked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) LOAD_HANDLER(r10, __end_interrupts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) cmpld r11,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) li r10,IMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) blt- 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* Test the soft mask state against our interrupt's bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) lbz r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 1: andi. r10,r10,IMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /* Associate vector numbers with bits in paca->irq_happened */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) .if IVEC == 0x500 || IVEC == 0xea0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) li r10,PACA_IRQ_EE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) .elseif IVEC == 0x900
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) li r10,PACA_IRQ_DEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) .elseif IVEC == 0xa00 || IVEC == 0xe80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) li r10,PACA_IRQ_DBELL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) .elseif IVEC == 0xe60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) li r10,PACA_IRQ_HMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) .elseif IVEC == 0xf00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) li r10,PACA_IRQ_PMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) .abort "Bad maskable vector"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) .if IHSRR_IF_HVMODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) bne masked_Hinterrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) bne masked_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) .elseif IHSRR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) bne masked_Hinterrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) bne masked_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) .if ISTACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) andi. r10,r12,MSR_PR /* See if coming from user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) 2: mr r10,r1 /* Save r1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) subi r1,r1,INT_FRAME_SIZE /* alloc frame on kernel stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) beq- 100f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ld r1,PACAKSAVE(r13) /* kernel stack to use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 100: tdgei r1,-INT_FRAME_SIZE /* trap if r1 is in userspace */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) std r9,_CCR(r1) /* save CR in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) std r11,_NIP(r1) /* save SRR0 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) std r12,_MSR(r1) /* save SRR1 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) std r10,0(r1) /* make stack chain pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) std r0,GPR0(r1) /* save r0 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) std r10,GPR1(r1) /* save r1 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) .if ISET_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) mtmsrd r10,1 /* Set MSR_RI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) .if ISTACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .if IKUAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) kuap_save_amr_and_lock r9, r10, cr1, cr0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) beq 101f /* if from kernel mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) ACCOUNT_CPU_USER_ENTRY(r13, r9, r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) std r9,_PPR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) 101:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) .if IKUAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) kuap_save_amr_and_lock r9, r10, cr1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /* Save original regs values from save area to stack frame. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ld r9,IAREA+EX_R9(r13) /* move r9, r10 to stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) ld r10,IAREA+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) std r9,GPR9(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) std r10,GPR10(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) ld r9,IAREA+EX_R11(r13) /* move r11 - r13 to stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) ld r10,IAREA+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ld r11,IAREA+EX_R13(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) std r9,GPR11(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) std r10,GPR12(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) std r11,GPR13(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) SAVE_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) .if IDAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) .if IISIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) ld r10,_NIP(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) ld r10,IAREA+EX_DAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) std r10,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) .if IDSISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) .if IISIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) ld r10,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) lis r11,DSISR_SRR1_MATCH_64S@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) and r10,r10,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) lwz r10,IAREA+EX_DSISR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) std r10,_DSISR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ld r10,IAREA+EX_CFAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) std r10,ORIG_GPR3(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) ld r10,IAREA+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) std r10,_CTR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) std r2,GPR2(r1) /* save r2 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) SAVE_4GPRS(3, r1) /* save r3 - r6 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) SAVE_2GPRS(7, r1) /* save r7, r8 in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) mflr r9 /* Get LR, later save to stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ld r2,PACATOC(r13) /* get kernel TOC into r2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) std r9,_LINK(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) lbz r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) mfspr r11,SPRN_XER /* save XER in stackframe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) std r10,SOFTE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) std r11,_XER(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) li r9,IVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) std r9,_TRAP(r1) /* set trap number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) li r10,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) ld r11,exception_marker@toc(r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) std r10,RESULT(r1) /* clear regs->result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) .if ISTACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ACCOUNT_STOLEN_TIME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) .if IRECONCILE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * On entry r13 points to the paca, r9-r13 are saved in the paca,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * SRR1, and relocation is on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * If stack=0, then the stack is already set in r1, and r1 is saved in r10.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * PPR save and CPU accounting is not done for the !stack case (XXX why not?)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) .macro GEN_COMMON name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) __GEN_COMMON_ENTRY \name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) __GEN_COMMON_BODY \name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * Restore all registers including H/SRR0/1 saved in a stack frame of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * standard exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) .macro EXCEPTION_RESTORE_REGS hsrr=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* Move original SRR0 and SRR1 into the respective regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ld r9,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) .if \hsrr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) mtspr SPRN_HSRR1,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) mtspr SPRN_SRR1,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) ld r9,_NIP(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) .if \hsrr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) mtspr SPRN_HSRR0,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) mtspr SPRN_SRR0,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) ld r9,_CTR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) mtctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) ld r9,_XER(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) mtxer r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) ld r9,_LINK(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) mtlr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) ld r9,_CCR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) mtcr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) REST_8GPRS(2, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) REST_4GPRS(10, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) REST_GPR(0, r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* restore original r1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) ld r1,GPR1(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) #define RUNLATCH_ON \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) BEGIN_FTR_SECTION \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) ld r3, PACA_THREAD_INFO(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) ld r4,TI_LOCAL_FLAGS(r3); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) andi. r0,r4,_TLF_RUNLATCH; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) beql ppc64_runlatch_on_trampoline; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) * When the idle code in power4_idle puts the CPU into NAP mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * it has to do so in a loop, and relies on the external interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * and decrementer interrupt entry code to get it out of the loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * to signal that it is in the loop and needs help to get out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #ifdef CONFIG_PPC_970_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) #define FINISH_NAP \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) BEGIN_FTR_SECTION \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ld r11, PACA_THREAD_INFO(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) ld r9,TI_LOCAL_FLAGS(r11); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) andi. r10,r9,_TLF_NAPPING; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) bnel power4_fixup_nap; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) #define FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * There are a few constraints to be concerned with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * - Real mode exceptions code/data must be located at their physical location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * - Virtual mode exceptions must be mapped at their 0xc000... location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * - Fixed location code must not call directly beyond the __end_interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * must be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * - LOAD_HANDLER targets must be within first 64K of physical 0 /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * virtual 0xc00...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * - Conditional branch targets must be within +/-32K of caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * therefore don't have to run in physically located code or rfid to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * virtual mode kernel code. However on relocatable kernels they do have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * to branch to KERNELBASE offset because the rest of the kernel (outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * the exception vectors) may be located elsewhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * Virtual exceptions correspond with physical, except their entry points
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * are offset by 0xc000000000000000 and also tend to get an added 0x4000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * offset applied. Virtual exceptions are enabled with the Alternate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * Interrupt Location (AIL) bit set in the LPCR. However this does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * guarantee they will be delivered virtually. Some conditions (see the ISA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * cause exceptions to be delivered in real mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * The scv instructions are a special case. They get a 0x3000 offset applied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * scv exceptions have unique reentrancy properties, see below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * It's impossible to receive interrupts below 0x300 via AIL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * KVM: None of the virtual exceptions are from the guest. Anything that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * escalated to HV=1 from HV=0 is delivered via real mode handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * We layout physical memory as follows:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * 0x0000 - 0x00ff : Secondary processor spin code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * 0x1900 - 0x2fff : Real mode trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) * 0x3000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * 0x5900 - 0x6fff : Relon mode trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * 0x7000 - 0x7fff : FWNMI data area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * 0x8000 - .... : Common interrupt handlers, remaining early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * setup code, rest of kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * vectors there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x3000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) OPEN_FIXED_SECTION(virt_vectors, 0x3000, 0x5900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) #ifdef CONFIG_PPC_POWERNV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) .globl start_real_trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) .globl end_real_trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) .globl start_virt_trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) .globl end_virt_trampolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * Data area reserved for FWNMI option.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * This address (0x7000) is fixed by the RPA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * pseries and powernv need to keep the whole page from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * 0x7000 to 0x8000 free for use by the firmware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) OPEN_TEXT_SECTION(0x8000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) OPEN_TEXT_SECTION(0x7000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) USE_FIXED_SECTION(real_vectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * This is the start of the interrupt handlers for pSeries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * This code runs with relocation off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * Code from here to __end_interrupts gets copied down to real
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * address 0x100 when we are running a relocatable kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * Therefore any relative branches in this section must only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * branch to labels in this section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) .globl __start_interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) __start_interrupts:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * Interrupt 0x3000 - System Call Vectored Interrupt (syscall).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * This is a synchronous interrupt invoked with the "scv" instruction. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * system call does not alter the HV bit, so it is directed to the OS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * scv instructions enter the kernel without changing EE, RI, ME, or HV.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) * In particular, this means we can take a maskable interrupt at any point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * in the scv handler, which is unlike any other interrupt. This is solved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * by treating the instruction addresses below __end_interrupts as being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * soft-masked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * AIL-0 mode scv exceptions go to 0x17000-0x17fff, but we set AIL-3 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * ensure scv is never executed with relocation off, which means AIL-0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * should never happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * Before leaving the below __end_interrupts text, at least of the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * must be true:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * - MSR[PR]=1 (i.e., return to userspace)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * - MSR_EE|MSR_RI is set (no reentrant exceptions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * - Standard kernel environment is set up (stack, paca, etc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * Call convention:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * syscall register convention is in Documentation/powerpc/syscall64-abi.rst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) EXC_VIRT_BEGIN(system_call_vectored, 0x3000, 0x1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /* SCV 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) mr r9,r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) mflr r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) mfctr r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) li r10,IRQS_ALL_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) #ifdef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) b system_call_vectored_tramp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) b system_call_vectored_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /* SCV 1 - 127 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) .rept 127
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mr r9,r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) mflr r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) mfctr r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) li r10,IRQS_ALL_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) li r0,-1 /* cause failure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) #ifdef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) b system_call_vectored_sigill_tramp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) b system_call_vectored_sigill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) EXC_VIRT_END(system_call_vectored, 0x3000, 0x1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) #ifdef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) TRAMP_VIRT_BEGIN(system_call_vectored_tramp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) __LOAD_HANDLER(r10, system_call_vectored_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) TRAMP_VIRT_BEGIN(system_call_vectored_sigill_tramp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) __LOAD_HANDLER(r10, system_call_vectored_sigill)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /* No virt vectors corresponding with 0x0..0x100 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) EXC_VIRT_NONE(0x4000, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Interrupt 0x100 - System Reset Interrupt (SRESET aka NMI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * This is a non-maskable, asynchronous interrupt always taken in real-mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * It is caused by:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * - Wake from power-saving state, on powernv.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * - An NMI from another CPU, triggered by firmware or hypercall.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * - As crash/debug signal injected from BMC, firmware or hypervisor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * Power-save wakeup is the only performance critical path, so this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * determined quickly as possible first. In this case volatile registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * can be discarded and SPRs like CFAR don't need to be read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * If not a powersave wakeup, then it's run as a regular interrupt, however
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) * it uses its own stack and PACA save area to preserve the regular kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * environment for debugging.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * This interrupt is not maskable, so triggering it when MSR[RI] is clear,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * or SCRATCH0 is in use, etc. may cause a crash. It's also not entirely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * correct to switch to virtual mode to run the regular interrupt handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * because it might be interrupted when the MMU is in a bad state (e.g., SLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * is clear).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * FWNMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * PAPR specifies a "fwnmi" facility which sends the sreset to a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * entry point with a different register set up. Some hypervisors will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * send the sreset to 0x100 in the guest if it is not fwnmi capable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * KVM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * Unlike most SRR interrupts, this may be taken by the host while executing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * in a guest, so a KVM test is required. KVM will pull the CPU out of guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * mode and then raise the sreset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) INT_DEFINE_BEGIN(system_reset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) IVEC=0x100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) IAREA=PACA_EXNMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) IVIRT=0 /* no virt entry point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * being used, so a nested NMI exception would corrupt it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) ISET_RI=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) ISTACK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) INT_DEFINE_END(system_reset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) EXC_REAL_BEGIN(system_reset, 0x100, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) #ifdef CONFIG_PPC_P7_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * If running native on arch 2.06 or later, check if we are waking up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) * from nap/sleep/winkle, and branch to idle handler. This tests SRR1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * bits 46:47. A non-0 value indicates that we are coming from a power
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * saving state. The idle wakeup handler initially runs in real mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * but we branch to the 0xc000... address so we can turn on relocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * with mtmsrd later, after SPRs are restored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Careful to minimise cost for the fast path (idle wakeup) while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * also avoiding clobbering CFAR for the debug path (non-idle).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * For the idle wake case volatile registers can be clobbered, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * is why we use those initially. If it turns out to not be an idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * wake, carefully put everything back the way it was, so we can use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * common exception macros to handle it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) SET_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) std r3,PACA_EXNMI+0*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) std r4,PACA_EXNMI+1*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) std r5,PACA_EXNMI+2*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) mfspr r3,SPRN_SRR1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) mfocrf r4,0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) rlwinm. r5,r3,47-31,30,31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) bne+ system_reset_idle_wake
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) /* Not powersave wakeup. Restore regs for regular interrupt handler. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) mtocrf 0x80,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) ld r3,PACA_EXNMI+0*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) ld r4,PACA_EXNMI+1*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) ld r5,PACA_EXNMI+2*8(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) GET_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) GEN_INT_ENTRY system_reset, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * In theory, we should not enable relocation here if it was disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * in SRR1, because the MMU may not be configured to support it (e.g.,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * SLB may have been cleared). In practice, there should only be a few
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * small windows where that's the case, and sreset is considered to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * be dangerous anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) EXC_REAL_END(system_reset, 0x100, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) EXC_VIRT_NONE(0x4100, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) #ifdef CONFIG_PPC_P7_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) TRAMP_REAL_BEGIN(system_reset_idle_wake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) /* We are waking up from idle, so may clobber any volatile register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) cmpwi cr1,r5,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) #ifdef CONFIG_PPC_PSERIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * Vectors for the FWNMI option. Share common code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) TRAMP_REAL_BEGIN(system_reset_fwnmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) GEN_INT_ENTRY system_reset, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) #endif /* CONFIG_PPC_PSERIES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) EXC_COMMON_BEGIN(system_reset_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) __GEN_COMMON_ENTRY system_reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) * to recover, but nested NMI will notice in_nmi and not recover
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * because of the use of the NMI stack. in_nmi reentrancy is tested in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * system_reset_exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) lhz r10,PACA_IN_NMI(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) addi r10,r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) sth r10,PACA_IN_NMI(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) mr r10,r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) ld r1,PACA_NMI_EMERG_SP(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) subi r1,r1,INT_FRAME_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) __GEN_COMMON_BODY system_reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * the right thing. We do not want to reconcile because that goes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) * through irq tracing which we don't want in NMI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * Save PACAIRQHAPPENED to RESULT (otherwise unused), and set HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * as we are running with MSR[EE]=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) li r10,IRQS_ALL_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) lbz r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) std r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) ori r10,r10,PACA_IRQ_HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) bl system_reset_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) /* Clear MSR_RI before setting SRR0 and SRR1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) li r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) mtmsrd r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * MSR_RI is clear, now we can decrement paca->in_nmi.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) lhz r10,PACA_IN_NMI(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) subi r10,r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) sth r10,PACA_IN_NMI(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * Restore soft mask settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) ld r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) ld r10,SOFTE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) kuap_restore_amr r9, r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) EXCEPTION_RESTORE_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) RFI_TO_USER_OR_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) GEN_KVM system_reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) * Interrupt 0x200 - Machine Check Interrupt (MCE).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) * This is a non-maskable interrupt always taken in real-mode. It can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * synchronous or asynchronous, caused by hardware or software, and it may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * taken in a power-saving state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * Similarly to system reset, this uses its own stack and PACA save area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * the difference is re-entrancy is allowed on the machine check stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * machine_check_early is run in real mode, and carefully decodes the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * machine check and tries to handle it (e.g., flush the SLB if there was an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * error detected there), determines if it was recoverable and logs the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * This early code does not "reconcile" irq soft-mask state like SRESET or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * regular interrupts do, so irqs_disabled() among other things may not work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * properly (irq disable/enable already doesn't work because irq tracing can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * not work in real mode).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * Then, depending on the execution context when the interrupt is taken, there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * are 3 main actions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * - Executing in kernel mode. The event is queued with irq_work, which means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * it is handled when it is next safe to do so (i.e., the kernel has enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * interrupts), which could be immediately when the interrupt returns. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * avoids nasty issues like switching to virtual mode when the MMU is in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * bad state, or when executing OPAL code. (SRESET is exposed to such issues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * but it has different priorities). Check to see if the CPU was in power
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * save, and return via the wake up code if it was.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * - Executing in user mode. machine_check_exception is run like a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * interrupt handler, which processes the data generated by the early handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * - Executing in guest mode. The interrupt is run with its KVM test, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * branches to KVM to deal with. KVM may queue the event for the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * to report later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * This interrupt is not maskable, so if it triggers when MSR[RI] is clear,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) * or SCRATCH0 is in use, it may cause a crash.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * KVM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) * See SRESET.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) INT_DEFINE_BEGIN(machine_check_early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) IVEC=0x200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) IAREA=PACA_EXMC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) IVIRT=0 /* no virt entry point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) IREALMODE_COMMON=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * MSR_RI is not enabled, because PACA_EXMC is being used, so a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * nested machine check corrupts it. machine_check_common enables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * MSR_RI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) ISET_RI=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) ISTACK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) INT_DEFINE_END(machine_check_early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) INT_DEFINE_BEGIN(machine_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) IVEC=0x200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) IAREA=PACA_EXMC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) IVIRT=0 /* no virt entry point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) ISET_RI=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) INT_DEFINE_END(machine_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) EXC_REAL_BEGIN(machine_check, 0x200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) GEN_INT_ENTRY machine_check_early, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) EXC_REAL_END(machine_check, 0x200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) EXC_VIRT_NONE(0x4200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) #ifdef CONFIG_PPC_PSERIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) TRAMP_REAL_BEGIN(machine_check_fwnmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) /* See comment at machine_check exception, don't turn on RI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) GEN_INT_ENTRY machine_check_early, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) #define MACHINE_CHECK_HANDLER_WINDUP \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) /* Clear MSR_RI before setting SRR0 and SRR1. */\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) li r9,0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) mtmsrd r9,1; /* Clear MSR_RI */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) /* Decrement paca->in_mce now RI is clear. */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) lhz r12,PACA_IN_MCE(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) subi r12,r12,1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) sth r12,PACA_IN_MCE(r13); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) EXCEPTION_RESTORE_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) EXC_COMMON_BEGIN(machine_check_early_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) __GEN_REALMODE_COMMON_ENTRY machine_check_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * Switch to mc_emergency stack and handle re-entrancy (we limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * the nested MCE upto level 4 to avoid stack overflow).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * Save MCE registers srr1, srr0, dar and dsisr and then set ME=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * We use paca->in_mce to check whether this is the first entry or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * nested machine check. We increment paca->in_mce to track nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * machine checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) * If this is the first entry then set stack pointer to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) * paca->mc_emergency_sp, otherwise r1 is already pointing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * stack frame on mc_emergency stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) * NOTE: We are here with MSR_ME=0 (off), which means we risk a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * checkstop if we get another machine check exception before we do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * rfid with MSR_ME=1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * This interrupt can wake directly from idle. If that is the case,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * the machine check is handled then the idle wakeup code is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) * to restore state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) lhz r10,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) cmpwi r10,0 /* Are we in nested machine check */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) cmpwi cr1,r10,MAX_MCE_DEPTH /* Are we at maximum nesting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) addi r10,r10,1 /* increment paca->in_mce */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) sth r10,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) mr r10,r1 /* Save r1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) bne 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) /* First machine check entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) 1: /* Limit nested MCE to level 4 to avoid stack overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) bgt cr1,unrecoverable_mce /* Check if we hit limit of 4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) __GEN_COMMON_BODY machine_check_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) bl enable_machine_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * system_reset_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) li r10,IRQS_ALL_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) lbz r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) std r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) ori r10,r10,PACA_IRQ_HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) bl machine_check_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) std r3,RESULT(r1) /* Save result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) ld r12,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * Restore soft mask settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) ld r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) ld r10,SOFTE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) #ifdef CONFIG_PPC_P7_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * Check if thread was in power saving mode. We come here when any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * of the following is true:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * a. thread wasn't in power saving mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * b. thread was in power saving mode with no state loss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * supervisor state loss or hypervisor state loss.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * Go back to nap/sleep/winkle mode again if (b) is true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) rlwinm. r11,r12,47-31,30,31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) bne machine_check_idle_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) * Check if we are coming from guest. If yes, then run the normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * exception handler which will take the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * to guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) lbz r11,HSTATE_IN_GUEST(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) cmpwi r11,0 /* Check if coming from guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) bne mce_deliver /* continue if we are. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * Check if we are coming from userspace. If yes, then run the normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * exception handler which will deliver the MC event to this kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) andi. r11,r12,MSR_PR /* See if coming from user. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) bne mce_deliver /* continue in V mode if we are. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) * At this point we are coming from kernel context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * Queue up the MCE event and return from the interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * But before that, check if this is an un-recoverable exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * If yes, then stay on emergency stack and panic.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) andi. r11,r12,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) beq unrecoverable_mce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) * Check if we have successfully handled/recovered from error, if not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) * then stay on emergency stack and panic.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) ld r3,RESULT(r1) /* Load result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) cmpdi r3,0 /* see if we handled MCE successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) beq unrecoverable_mce /* if !handled then panic */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) * Return from MC interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * Queue up the MCE event so that we can log it later, while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) * returning from kernel or opal call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) bl machine_check_queue_event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) MACHINE_CHECK_HANDLER_WINDUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) mce_deliver:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) * This is a host user or guest MCE. Restore all registers, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * run the "late" handler. For host user, this will run the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * machine_check_exception handler in virtual mode like a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) * interrupt handler. For guest, this will trigger the KVM test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * and branch to the KVM interrupt similarly to other interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) ld r10,ORIG_GPR3(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) mtspr SPRN_CFAR,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) MACHINE_CHECK_HANDLER_WINDUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) GEN_INT_ENTRY machine_check, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) EXC_COMMON_BEGIN(machine_check_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * Machine check is different because we use a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * save area: PACA_EXMC instead of PACA_EXGEN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) GEN_COMMON machine_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) /* Enable MSR_RI when finished with PACA_EXMC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) bl machine_check_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) GEN_KVM machine_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) #ifdef CONFIG_PPC_P7_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) * This is an idle wakeup. Low level machine check has already been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) * done. Queue the event then call the idle code to do the wake up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) EXC_COMMON_BEGIN(machine_check_idle_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) bl machine_check_queue_event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) * GPR-loss wakeups are relatively straightforward, because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) * idle sleep code has saved all non-volatile registers on its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) * own stack, and r1 in PACAR1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * For no-loss wakeups the r1 and lr registers used by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) * early machine check handler have to be restored first. r2 is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * the kernel TOC, so no need to restore it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * Then decrement MCE nesting after finishing with the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) ld r3,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) ld r4,_LINK(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) ld r1,GPR1(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) lhz r11,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) subi r11,r11,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) sth r11,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) mtlr r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) rlwinm r10,r3,47-31,30,31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) cmpwi cr1,r10,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) b idle_return_gpr_loss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) EXC_COMMON_BEGIN(unrecoverable_mce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * We are going down. But there are chances that we might get hit by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * another MCE during panic path and we may run into unstable state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * with no way out. Hence, turn ME bit off while going down, so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * when another MCE is hit during panic path, system will checkstop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * and hypervisor will get restarted cleanly by SP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) li r10,0 /* clear MSR_RI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) bl disable_machine_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) ld r10,PACAKMSR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) li r3,MSR_ME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) andc r10,r10,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) mtmsrd r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) lhz r12,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) subi r12,r12,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) sth r12,PACA_IN_MCE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /* Invoke machine_check_exception to print MCE event and panic. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) bl machine_check_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) * We will not reach here. Even if we did, there is no way out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) * Call unrecoverable_exception and die.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) bl unrecoverable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) b .
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) * Interrupt 0x300 - Data Storage Interrupt (DSI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * This is a synchronous interrupt generated due to a data access exception,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) * e.g., a load orstore which does not have a valid page table entry with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) * permissions. DAWR matches also fault here, as do RC updates, and minor misc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) * errors e.g., copy/paste, AMO, certain invalid CI accesses, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) * - Hash MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) * Go to do_hash_page first to see if the HPT can be filled from an entry in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * the Linux page table. Hash faults can hit in kernel mode in a fairly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * arbitrary state (e.g., interrupts disabled, locks held) when accessing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) * "non-bolted" regions, e.g., vmalloc space. However these should always be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) * backed by Linux page tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) * If none is found, do a Linux page fault. Linux page faults can happen in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) * kernel mode due to user copy operations of course.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) * MMU context, which may cause a DSI in the host, which must go to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * KVM handler. MSR[IR] is not enabled, so the real-mode handler will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * always be used regardless of AIL setting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) * - Radix MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * The hardware loads from the Linux page table directly, so a fault goes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * immediately to Linux page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) * Conditions like DAWR match are handled on the way in to Linux page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) INT_DEFINE_BEGIN(data_access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) IVEC=0x300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) INT_DEFINE_END(data_access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) EXC_REAL_BEGIN(data_access, 0x300, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) GEN_INT_ENTRY data_access, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) EXC_REAL_END(data_access, 0x300, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) GEN_INT_ENTRY data_access, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) EXC_VIRT_END(data_access, 0x4300, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) EXC_COMMON_BEGIN(data_access_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) GEN_COMMON data_access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) ld r5,_DSISR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) BEGIN_MMU_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) ld r6,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) li r3,0x300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) b do_hash_page /* Try to handle as hpte fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) MMU_FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) b handle_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) GEN_KVM data_access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) * Interrupt 0x380 - Data Segment Interrupt (DSLB).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) * This is a synchronous interrupt in response to an MMU fault missing SLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) * entry for HPT, or an address outside RPT translation range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) * - HPT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) * This refills the SLB, or reports an access fault similarly to a bad page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) * fault. When coming from user-mode, the SLB handler may access any kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) * data, though it may itself take a DSLB. When coming from kernel mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) * recursive faults must be avoided so access is restricted to the kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) * image text/data, kernel stack, and any data allocated below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) * ppc64_bolted_size (first segment). The kernel handler must avoid stomping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) * on user-handler data structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * KVM: Same as 0x300, DSLB must test for KVM guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) * A dedicated save area EXSLB is used (XXX: but it actually need not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) * these days, we could use EXGEN).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) INT_DEFINE_BEGIN(data_access_slb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) IVEC=0x380
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) IAREA=PACA_EXSLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) INT_DEFINE_END(data_access_slb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) GEN_INT_ENTRY data_access_slb, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) EXC_REAL_END(data_access_slb, 0x380, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) GEN_INT_ENTRY data_access_slb, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) EXC_COMMON_BEGIN(data_access_slb_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) GEN_COMMON data_access_slb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) BEGIN_MMU_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) /* HPT case, do SLB fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) bl do_slb_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) cmpdi r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) bne- 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) b fast_interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) 1: /* Error case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) MMU_FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) /* Radix case, access is outside page table range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) li r3,-EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) std r3,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) ld r5,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) bl do_bad_slb_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) GEN_KVM data_access_slb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) * Interrupt 0x400 - Instruction Storage Interrupt (ISI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) * This is a synchronous interrupt in response to an MMU fault due to an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) * instruction fetch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * Similar to DSI, though in response to fetch. The faulting address is found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * in SRR0 (rather than DAR), and status in SRR1 (rather than DSISR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) INT_DEFINE_BEGIN(instruction_access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) IVEC=0x400
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) IISIDE=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) INT_DEFINE_END(instruction_access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) EXC_REAL_BEGIN(instruction_access, 0x400, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) GEN_INT_ENTRY instruction_access, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) EXC_REAL_END(instruction_access, 0x400, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) GEN_INT_ENTRY instruction_access, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) EXC_VIRT_END(instruction_access, 0x4400, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) EXC_COMMON_BEGIN(instruction_access_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) GEN_COMMON instruction_access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) ld r5,_DSISR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) BEGIN_MMU_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) ld r6,_MSR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) li r3,0x400
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) b do_hash_page /* Try to handle as hpte fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) MMU_FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) b handle_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) GEN_KVM instruction_access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) * Interrupt 0x480 - Instruction Segment Interrupt (ISLB).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) * This is a synchronous interrupt in response to an MMU fault due to an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) * instruction fetch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) * Similar to DSLB, though in response to fetch. The faulting address is found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) * in SRR0 (rather than DAR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) INT_DEFINE_BEGIN(instruction_access_slb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) IVEC=0x480
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) IAREA=PACA_EXSLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) IISIDE=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) INT_DEFINE_END(instruction_access_slb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) GEN_INT_ENTRY instruction_access_slb, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) GEN_INT_ENTRY instruction_access_slb, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) EXC_COMMON_BEGIN(instruction_access_slb_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) GEN_COMMON instruction_access_slb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) BEGIN_MMU_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) /* HPT case, do SLB fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) bl do_slb_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) cmpdi r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) bne- 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) b fast_interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) 1: /* Error case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) MMU_FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) /* Radix case, access is outside page table range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) li r3,-EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) std r3,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) ld r5,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) bl do_bad_slb_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) GEN_KVM instruction_access_slb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * Interrupt 0x500 - External Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) * This is an asynchronous maskable interrupt in response to an "external
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * exception" from the interrupt controller or hypervisor (e.g., device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) * interrupt). It is maskable in hardware by clearing MSR[EE], and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) * soft-maskable with IRQS_DISABLED mask (i.e., local_irq_disable()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) * When running in HV mode, Linux sets up the LPCR[LPES] bit such that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) * interrupts are delivered with HSRR registers, guests use SRRs, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) * reqiures IHSRR_IF_HVMODE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) * On bare metal POWER9 and later, Linux sets the LPCR[HVICE] bit such that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) * external interrupts are delivered as Hypervisor Virtualization Interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) * rather than External Interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) * This calls into Linux IRQ handler. NVGPRs are not saved to reduce overhead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) * because registers at the time of the interrupt are not so important as it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) * asynchronous.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) * If soft masked, the masked handler will note the pending interrupt for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * replay, and clear MSR[EE] in the interrupted context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) INT_DEFINE_BEGIN(hardware_interrupt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) IVEC=0x500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) IHSRR_IF_HVMODE=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) INT_DEFINE_END(hardware_interrupt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) GEN_INT_ENTRY hardware_interrupt, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) GEN_INT_ENTRY hardware_interrupt, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) EXC_COMMON_BEGIN(hardware_interrupt_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) GEN_COMMON hardware_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) bl do_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) GEN_KVM hardware_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * Interrupt 0x600 - Alignment Interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * This is a synchronous interrupt in response to data alignment fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) INT_DEFINE_BEGIN(alignment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) IVEC=0x600
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) INT_DEFINE_END(alignment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) EXC_REAL_BEGIN(alignment, 0x600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) GEN_INT_ENTRY alignment, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) EXC_REAL_END(alignment, 0x600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) GEN_INT_ENTRY alignment, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) EXC_VIRT_END(alignment, 0x4600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) EXC_COMMON_BEGIN(alignment_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) GEN_COMMON alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) bl alignment_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) REST_NVGPRS(r1) /* instruction emulation may change GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) GEN_KVM alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) * Interrupt 0x700 - Program Interrupt (program check).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) * This is a synchronous interrupt in response to various instruction faults:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) * traps, privilege errors, TM errors, floating point exceptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) * This interrupt may use the "emergency stack" in some cases when being taken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) * from kernel context, which complicates handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) INT_DEFINE_BEGIN(program_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) IVEC=0x700
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) INT_DEFINE_END(program_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) EXC_REAL_BEGIN(program_check, 0x700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) GEN_INT_ENTRY program_check, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) EXC_REAL_END(program_check, 0x700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) EXC_VIRT_BEGIN(program_check, 0x4700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) GEN_INT_ENTRY program_check, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) EXC_VIRT_END(program_check, 0x4700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) EXC_COMMON_BEGIN(program_check_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) __GEN_COMMON_ENTRY program_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) * It's possible to receive a TM Bad Thing type program check with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) * userspace register values (in particular r1), but with SRR1 reporting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) * that we came from the kernel. Normally that would confuse the bad
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) * stack logic, and we would report a bad kernel stack pointer. Instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) * we switch to the emergency stack if we're taking a TM Bad Thing from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) * the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) andi. r10,r12,MSR_PR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) bne .Lnormal_stack /* If userspace, go normal path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) andis. r10,r12,(SRR1_PROGTM)@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) bne .Lemergency_stack /* If TM, emergency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) blt .Lnormal_stack /* normal path if not */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) /* Use the emergency stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) .Lemergency_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) andi. r10,r12,MSR_PR /* Set CR0 correctly for label */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) /* 3 in EXCEPTION_PROLOG_COMMON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) mr r10,r1 /* Save r1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) ld r1,PACAEMERGSP(r13) /* Use emergency stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) __ISTACK(program_check)=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) __GEN_COMMON_BODY program_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) b .Ldo_program_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) .Lnormal_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) __ISTACK(program_check)=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) __GEN_COMMON_BODY program_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) .Ldo_program_check:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) bl program_check_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) REST_NVGPRS(r1) /* instruction emulation may change GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) GEN_KVM program_check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) * Interrupt 0x800 - Floating-Point Unavailable Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) * This is a synchronous interrupt in response to executing an fp instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) * with MSR[FP]=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) * This will load FP registers and enable the FP bit if coming from userspace,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) * otherwise report a bad kernel use of FP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) INT_DEFINE_BEGIN(fp_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) IVEC=0x800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) INT_DEFINE_END(fp_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) EXC_REAL_BEGIN(fp_unavailable, 0x800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) GEN_INT_ENTRY fp_unavailable, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) EXC_REAL_END(fp_unavailable, 0x800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) GEN_INT_ENTRY fp_unavailable, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) EXC_COMMON_BEGIN(fp_unavailable_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) GEN_COMMON fp_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) bne 1f /* if from user, just load it up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) bl kernel_fp_unavailable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) 0: trap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) EMIT_BUG_ENTRY 0b, __FILE__, __LINE__, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) * transaction), go do TM stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) bne- 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) END_FTR_SECTION_IFSET(CPU_FTR_TM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) bl load_up_fpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) b fast_interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) 2: /* User process was in a transaction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) bl fp_unavailable_tm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) GEN_KVM fp_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) * Interrupt 0x900 - Decrementer Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) * This is an asynchronous interrupt in response to a decrementer exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) * (e.g., DEC has wrapped below zero). It is maskable in hardware by clearing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) * MSR[EE], and soft-maskable with IRQS_DISABLED mask (i.e.,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) * local_irq_disable()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) * This calls into Linux timer handler. NVGPRs are not saved (see 0x500).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) * If soft masked, the masked handler will note the pending interrupt for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) * replay, and bump the decrementer to a high value, leaving MSR[EE] enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) * in the interrupted context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) * If PPC_WATCHDOG is configured, the soft masked handler will actually set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) * things back up to run soft_nmi_interrupt as a regular interrupt handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) * on the emergency stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) INT_DEFINE_BEGIN(decrementer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) IVEC=0x900
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) INT_DEFINE_END(decrementer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) EXC_REAL_BEGIN(decrementer, 0x900, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) GEN_INT_ENTRY decrementer, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) EXC_REAL_END(decrementer, 0x900, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) GEN_INT_ENTRY decrementer, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) EXC_VIRT_END(decrementer, 0x4900, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) EXC_COMMON_BEGIN(decrementer_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) GEN_COMMON decrementer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) bl timer_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) GEN_KVM decrementer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * Interrupt 0x980 - Hypervisor Decrementer Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) * This is an asynchronous interrupt, similar to 0x900 but for the HDEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) * register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) * Linux does not use this outside KVM where it's used to keep a host timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) * while the guest is given control of DEC. It should normally be caught by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * the KVM test and routed there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) INT_DEFINE_BEGIN(hdecrementer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) IVEC=0x980
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) ISTACK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) INT_DEFINE_END(hdecrementer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) EXC_REAL_BEGIN(hdecrementer, 0x980, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) GEN_INT_ENTRY hdecrementer, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) EXC_REAL_END(hdecrementer, 0x980, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) GEN_INT_ENTRY hdecrementer, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) EXC_COMMON_BEGIN(hdecrementer_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) __GEN_COMMON_ENTRY hdecrementer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * Hypervisor decrementer interrupts not caught by the KVM test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) * shouldn't occur but are sometimes left pending on exit from a KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) * guest. We don't need to do anything to clear them, as they are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) * edge-triggered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) * Be careful to avoid touching the kernel stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) ld r10,PACA_EXGEN+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) mtcrf 0x80,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) ld r9,PACA_EXGEN+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) ld r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) ld r11,PACA_EXGEN+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) ld r12,PACA_EXGEN+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) ld r13,PACA_EXGEN+EX_R13(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) HRFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) GEN_KVM hdecrementer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) * This is an asynchronous interrupt in response to a msgsndp doorbell.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) * IRQS_DISABLED mask (i.e., local_irq_disable()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) * Guests may use this for IPIs between threads in a core if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) * hypervisor supports it. NVGPRS are not saved (see 0x500).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) * If soft masked, the masked handler will note the pending interrupt for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) * replay, leaving MSR[EE] enabled in the interrupted context because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) * doorbells are edge triggered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) INT_DEFINE_BEGIN(doorbell_super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) IVEC=0xa00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) INT_DEFINE_END(doorbell_super)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) EXC_REAL_BEGIN(doorbell_super, 0xa00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) GEN_INT_ENTRY doorbell_super, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) EXC_REAL_END(doorbell_super, 0xa00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) GEN_INT_ENTRY doorbell_super, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) EXC_COMMON_BEGIN(doorbell_super_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) GEN_COMMON doorbell_super
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) #ifdef CONFIG_PPC_DOORBELL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) bl doorbell_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) GEN_KVM doorbell_super
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) EXC_REAL_NONE(0xb00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) EXC_VIRT_NONE(0x4b00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) * Interrupt 0xc00 - System Call Interrupt (syscall, hcall).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) * This is a synchronous interrupt invoked with the "sc" instruction. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) * system call is invoked with "sc 0" and does not alter the HV bit, so it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) * is directed to the currently running OS. The hypercall is invoked with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) * "sc 1" and it sets HV=1, so it elevates to hypervisor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) * In HPT, sc 1 always goes to 0xc00 real mode. In RADIX, sc 1 can go to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) * 0x4c00 virtual mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) * If the KVM test fires then it was due to a hypercall and is accordingly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) * routed to KVM. Otherwise this executes a normal Linux system call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) * Call convention:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * syscall and hypercalls register conventions are documented in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) * Documentation/powerpc/syscall64-abi.rst and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) * Documentation/powerpc/papr_hcalls.rst respectively.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) * The intersection of volatile registers that don't contain possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) * inputs is: cr0, xer, ctr. We may use these as scratch regs upon entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) * without saving, though xer is not a good idea to use, as hardware may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) * interpret some bits so it may be costly to change them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) INT_DEFINE_BEGIN(system_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) IVEC=0xc00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) INT_DEFINE_END(system_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) .macro SYSTEM_CALL virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) * There is a little bit of juggling to get syscall and hcall
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) * working well. Save r13 in ctr to avoid using SPRG scratch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) * register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) * Userspace syscalls have already saved the PPR, hcalls must save
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) * it before setting HMT_MEDIUM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) mtctr r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) std r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) INTERRUPT_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) KVMTEST system_call /* uses r10, branch to system_call_kvm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) mfctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) mr r9,r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) GET_PACA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) INTERRUPT_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) cmpdi r0,0x1ebe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) beq- 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) /* We reach here with PACA in r13, r13 in r9. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) mfspr r11,SPRN_SRR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) mfspr r12,SPRN_SRR1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) HMT_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) .if ! \virt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) __LOAD_HANDLER(r10, system_call_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) mtspr SPRN_SRR0,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) ld r10,PACAKMSR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) mtspr SPRN_SRR1,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) b . /* prevent speculative execution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) mtmsrd r10,1 /* Set RI (EE=0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) #ifdef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) __LOAD_HANDLER(r10, system_call_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) b system_call_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) /* Fast LE/BE switch system call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) 1: mfspr r12,SPRN_SRR1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) xori r12,r12,MSR_LE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) mtspr SPRN_SRR1,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) mr r13,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) RFI_TO_USER /* return to userspace */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) b . /* prevent speculative execution */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) SYSTEM_CALL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) EXC_REAL_END(system_call, 0xc00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) SYSTEM_CALL 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) EXC_VIRT_END(system_call, 0x4c00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) TRAMP_REAL_BEGIN(system_call_kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) * This is a hcall, so register convention is as above, with these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) * differences:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) * r13 = PACA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) * ctr = orig r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * orig r10 saved in PACA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) * Save the PPR (on systems that support it) before changing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * HMT_MEDIUM. That allows the KVM code to save that value into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) * guest state (it is the guest's PPR value).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) mfspr r10,SPRN_PPR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) std r10,HSTATE_PPR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) HMT_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) mfctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) SET_SCRATCH0(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) mfcr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) std r12,HSTATE_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) sldi r12,r10,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) ori r12,r12,0xc00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) #ifdef CONFIG_RELOCATABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) * outside the head section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) __LOAD_FAR_HANDLER(r10, kvmppc_interrupt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) ld r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) bctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) ld r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) b kvmppc_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) * Interrupt 0xd00 - Trace Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) * This is a synchronous interrupt in response to instruction step or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) * breakpoint faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) INT_DEFINE_BEGIN(single_step)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) IVEC=0xd00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) INT_DEFINE_END(single_step)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) EXC_REAL_BEGIN(single_step, 0xd00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) GEN_INT_ENTRY single_step, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) EXC_REAL_END(single_step, 0xd00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) GEN_INT_ENTRY single_step, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) EXC_VIRT_END(single_step, 0x4d00, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) EXC_COMMON_BEGIN(single_step_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) GEN_COMMON single_step
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) bl single_step_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) GEN_KVM single_step
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) * This is a synchronous interrupt in response to an MMU fault caused by a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) * guest data access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) * This should always get routed to KVM. In radix MMU mode, this is caused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * by a guest nested radix access that can't be performed due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) * partition scope page table. In hash mode, this can be caused by guests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) * running with translation disabled (virtual real mode) or with VPM enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) * KVM will update the page table structures or disallow the access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) INT_DEFINE_BEGIN(h_data_storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) IVEC=0xe00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) IDAR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) IDSISR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) INT_DEFINE_END(h_data_storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) EXC_REAL_BEGIN(h_data_storage, 0xe00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) GEN_INT_ENTRY h_data_storage, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) EXC_REAL_END(h_data_storage, 0xe00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) GEN_INT_ENTRY h_data_storage, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) EXC_VIRT_END(h_data_storage, 0x4e00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) EXC_COMMON_BEGIN(h_data_storage_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) GEN_COMMON h_data_storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) BEGIN_MMU_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) li r5,SIGSEGV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) bl bad_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) MMU_FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) GEN_KVM h_data_storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) * This is a synchronous interrupt in response to an MMU fault caused by a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) * guest instruction fetch, similar to HDSI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) INT_DEFINE_BEGIN(h_instr_storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) IVEC=0xe20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) INT_DEFINE_END(h_instr_storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) EXC_REAL_BEGIN(h_instr_storage, 0xe20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) GEN_INT_ENTRY h_instr_storage, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) EXC_REAL_END(h_instr_storage, 0xe20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) GEN_INT_ENTRY h_instr_storage, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) EXC_COMMON_BEGIN(h_instr_storage_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) GEN_COMMON h_instr_storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) GEN_KVM h_instr_storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) INT_DEFINE_BEGIN(emulation_assist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) IVEC=0xe40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) INT_DEFINE_END(emulation_assist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) EXC_REAL_BEGIN(emulation_assist, 0xe40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) GEN_INT_ENTRY emulation_assist, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) EXC_REAL_END(emulation_assist, 0xe40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) GEN_INT_ENTRY emulation_assist, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) EXC_VIRT_END(emulation_assist, 0x4e40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) EXC_COMMON_BEGIN(emulation_assist_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) GEN_COMMON emulation_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) bl emulation_assist_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) REST_NVGPRS(r1) /* instruction emulation may change GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) GEN_KVM emulation_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) * This is an asynchronous interrupt caused by a Hypervisor Maintenance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) * Exception. It is always taken in real mode but uses HSRR registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) * unlike SRESET and MCE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) * It is maskable in hardware by clearing MSR[EE], and partially soft-maskable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) * with IRQS_DISABLED mask (i.e., local_irq_disable()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) * This is a special case, this is handled similarly to machine checks, with an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) * initial real mode handler that is not soft-masked, which attempts to fix the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) * problem. Then a regular handler which is soft-maskable and reports the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) * problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) * The emergency stack is used for the early real mode handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) * XXX: unclear why MCE and HMI schemes could not be made common, e.g.,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) * either use soft-masking for the MCE, or use irq_work for the HMI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) * KVM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) * Unlike MCE, this calls into KVM without calling the real mode handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) * first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) INT_DEFINE_BEGIN(hmi_exception_early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) IVEC=0xe60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) IREALMODE_COMMON=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) ISTACK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) INT_DEFINE_END(hmi_exception_early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) INT_DEFINE_BEGIN(hmi_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) IVEC=0xe60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) INT_DEFINE_END(hmi_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) EXC_REAL_END(hmi_exception, 0xe60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) EXC_VIRT_NONE(0x4e60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) EXC_COMMON_BEGIN(hmi_exception_early_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) __GEN_REALMODE_COMMON_ENTRY hmi_exception_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) mr r10,r1 /* Save r1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) __GEN_COMMON_BODY hmi_exception_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) bl hmi_exception_realmode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) cmpdi cr0,r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) bne 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) EXCEPTION_RESTORE_REGS hsrr=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) HRFI_TO_USER_OR_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) * Go to virtual mode and pull the HMI event information from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) * firmware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) EXCEPTION_RESTORE_REGS hsrr=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) GEN_INT_ENTRY hmi_exception, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) GEN_KVM hmi_exception_early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) EXC_COMMON_BEGIN(hmi_exception_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) GEN_COMMON hmi_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) bl handle_hmi_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) GEN_KVM hmi_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) * This is an asynchronous interrupt in response to a msgsnd doorbell.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) * Similar to the 0xa00 doorbell but for host rather than guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) INT_DEFINE_BEGIN(h_doorbell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) IVEC=0xe80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) INT_DEFINE_END(h_doorbell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) EXC_REAL_BEGIN(h_doorbell, 0xe80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) GEN_INT_ENTRY h_doorbell, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) EXC_REAL_END(h_doorbell, 0xe80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) GEN_INT_ENTRY h_doorbell, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) EXC_COMMON_BEGIN(h_doorbell_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) GEN_COMMON h_doorbell
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) #ifdef CONFIG_PPC_DOORBELL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) bl doorbell_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) GEN_KVM h_doorbell
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) * Interrupt 0xea0 - Hypervisor Virtualization Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) * This is an asynchronous interrupt in response to an "external exception".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) * Similar to 0x500 but for host only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) INT_DEFINE_BEGIN(h_virt_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) IVEC=0xea0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) IMASK=IRQS_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) INT_DEFINE_END(h_virt_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) EXC_REAL_BEGIN(h_virt_irq, 0xea0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) GEN_INT_ENTRY h_virt_irq, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) EXC_REAL_END(h_virt_irq, 0xea0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) GEN_INT_ENTRY h_virt_irq, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) EXC_COMMON_BEGIN(h_virt_irq_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) GEN_COMMON h_virt_irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) bl do_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) GEN_KVM h_virt_irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) EXC_REAL_NONE(0xec0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) EXC_VIRT_NONE(0x4ec0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) EXC_REAL_NONE(0xee0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) EXC_VIRT_NONE(0x4ee0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) * Interrupt 0xf00 - Performance Monitor Interrupt (PMI, PMU).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) * This is an asynchronous interrupt in response to a PMU exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) * It is maskable in hardware by clearing MSR[EE], and soft-maskable with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) * IRQS_PMI_DISABLED mask (NOTE: NOT local_irq_disable()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) * This calls into the perf subsystem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) * Like the watchdog soft-nmi, it appears an NMI interrupt to Linux, in that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) * runs under local_irq_disable. However it may be soft-masked in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) * powerpc-specific code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) * If soft masked, the masked handler will note the pending interrupt for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) * replay, and clear MSR[EE] in the interrupted context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) INT_DEFINE_BEGIN(performance_monitor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) IVEC=0xf00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) IMASK=IRQS_PMI_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) INT_DEFINE_END(performance_monitor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) EXC_REAL_BEGIN(performance_monitor, 0xf00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) GEN_INT_ENTRY performance_monitor, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) EXC_REAL_END(performance_monitor, 0xf00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) GEN_INT_ENTRY performance_monitor, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) EXC_COMMON_BEGIN(performance_monitor_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) GEN_COMMON performance_monitor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) FINISH_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) RUNLATCH_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) bl performance_monitor_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) GEN_KVM performance_monitor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) * Interrupt 0xf20 - Vector Unavailable Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) * This is a synchronous interrupt in response to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) * executing a vector (or altivec) instruction with MSR[VEC]=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) * Similar to FP unavailable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) INT_DEFINE_BEGIN(altivec_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) IVEC=0xf20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) INT_DEFINE_END(altivec_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) EXC_REAL_BEGIN(altivec_unavailable, 0xf20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) GEN_INT_ENTRY altivec_unavailable, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) EXC_REAL_END(altivec_unavailable, 0xf20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) EXC_COMMON_BEGIN(altivec_unavailable_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) GEN_COMMON altivec_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) #ifdef CONFIG_ALTIVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) beq 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) BEGIN_FTR_SECTION_NESTED(69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) * transaction), go do TM stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) bne- 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) bl load_up_altivec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) b fast_interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) 2: /* User process was in a transaction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) bl altivec_unavailable_tm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) bl altivec_unavailable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) GEN_KVM altivec_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) * Interrupt 0xf40 - VSX Unavailable Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) * This is a synchronous interrupt in response to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) * executing a VSX instruction with MSR[VSX]=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) * Similar to FP unavailable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) INT_DEFINE_BEGIN(vsx_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) IVEC=0xf40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) IRECONCILE=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) INT_DEFINE_END(vsx_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) EXC_REAL_BEGIN(vsx_unavailable, 0xf40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) GEN_INT_ENTRY vsx_unavailable, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) EXC_REAL_END(vsx_unavailable, 0xf40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) EXC_COMMON_BEGIN(vsx_unavailable_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) GEN_COMMON vsx_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) #ifdef CONFIG_VSX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) beq 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) BEGIN_FTR_SECTION_NESTED(69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) /* Test if 2 TM state bits are zero. If non-zero (ie. userspace was in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) * transaction), go do TM stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) rldicl. r0, r12, (64-MSR_TS_LG), (64-2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) bne- 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) END_FTR_SECTION_NESTED(CPU_FTR_TM, CPU_FTR_TM, 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) b load_up_vsx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) 2: /* User process was in a transaction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) bl vsx_unavailable_tm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) END_FTR_SECTION_IFSET(CPU_FTR_VSX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) RECONCILE_IRQ_STATE(r10, r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) bl vsx_unavailable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) GEN_KVM vsx_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) * Interrupt 0xf60 - Facility Unavailable Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) * This is a synchronous interrupt in response to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) * executing an instruction without access to the facility that can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) * resolved by the OS (e.g., FSCR, MSR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) * Similar to FP unavailable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) INT_DEFINE_BEGIN(facility_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) IVEC=0xf60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) INT_DEFINE_END(facility_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) EXC_REAL_BEGIN(facility_unavailable, 0xf60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) GEN_INT_ENTRY facility_unavailable, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) EXC_REAL_END(facility_unavailable, 0xf60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) GEN_INT_ENTRY facility_unavailable, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) EXC_COMMON_BEGIN(facility_unavailable_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) GEN_COMMON facility_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) bl facility_unavailable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) REST_NVGPRS(r1) /* instruction emulation may change GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) GEN_KVM facility_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) * This is a synchronous interrupt in response to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) * executing an instruction without access to the facility that can only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) * be resolved in HV mode (e.g., HFSCR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) * Similar to FP unavailable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) INT_DEFINE_BEGIN(h_facility_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) IVEC=0xf80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) IKVM_VIRT=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) INT_DEFINE_END(h_facility_unavailable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) EXC_REAL_BEGIN(h_facility_unavailable, 0xf80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) GEN_INT_ENTRY h_facility_unavailable, virt=0, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) EXC_COMMON_BEGIN(h_facility_unavailable_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) GEN_COMMON h_facility_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) bl facility_unavailable_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) GEN_KVM h_facility_unavailable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) EXC_REAL_NONE(0xfa0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) EXC_VIRT_NONE(0x4fa0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) EXC_REAL_NONE(0xfc0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) EXC_VIRT_NONE(0x4fc0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) EXC_REAL_NONE(0xfe0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) EXC_VIRT_NONE(0x4fe0, 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) EXC_REAL_NONE(0x1000, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) EXC_VIRT_NONE(0x5000, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) EXC_REAL_NONE(0x1100, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) EXC_VIRT_NONE(0x5100, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) #ifdef CONFIG_CBE_RAS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) INT_DEFINE_BEGIN(cbe_system_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) IVEC=0x1200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) INT_DEFINE_END(cbe_system_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) GEN_INT_ENTRY cbe_system_error, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) EXC_REAL_END(cbe_system_error, 0x1200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) EXC_VIRT_NONE(0x5200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) EXC_COMMON_BEGIN(cbe_system_error_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) GEN_COMMON cbe_system_error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) bl cbe_system_error_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) GEN_KVM cbe_system_error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) #else /* CONFIG_CBE_RAS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) EXC_REAL_NONE(0x1200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) EXC_VIRT_NONE(0x5200, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) INT_DEFINE_BEGIN(instruction_breakpoint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) IVEC=0x1300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) INT_DEFINE_END(instruction_breakpoint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) EXC_REAL_BEGIN(instruction_breakpoint, 0x1300, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) GEN_INT_ENTRY instruction_breakpoint, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) GEN_INT_ENTRY instruction_breakpoint, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) EXC_COMMON_BEGIN(instruction_breakpoint_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) GEN_COMMON instruction_breakpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) bl instruction_breakpoint_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) GEN_KVM instruction_breakpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) EXC_REAL_NONE(0x1400, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) EXC_VIRT_NONE(0x5400, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) * Interrupt 0x1500 - Soft Patch Interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) * Handling:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) * This is an implementation specific interrupt which can be used for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) * range of exceptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) * This interrupt handler is unique in that it runs the denormal assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) * code even for guests (and even in guest context) without going to KVM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) * for speed. POWER9 does not raise denorm exceptions, so this special case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) * could be phased out in future to reduce special cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) INT_DEFINE_BEGIN(denorm_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) IVEC=0x1500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) IBRANCH_TO_COMMON=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) INT_DEFINE_END(denorm_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) GEN_INT_ENTRY denorm_exception, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) #ifdef CONFIG_PPC_DENORMALISATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) bne+ denorm_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) GEN_BRANCH_TO_COMMON denorm_exception, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) EXC_REAL_END(denorm_exception, 0x1500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) #ifdef CONFIG_PPC_DENORMALISATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) GEN_INT_ENTRY denorm_exception, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) andis. r10,r12,(HSRR1_DENORM)@h /* denorm? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) bne+ denorm_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) GEN_BRANCH_TO_COMMON denorm_exception, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) EXC_VIRT_NONE(0x5500, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) #ifdef CONFIG_PPC_DENORMALISATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) TRAMP_REAL_BEGIN(denorm_assist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) * To denormalise we need to move a copy of the register to itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) * For POWER6 do that here for all FP regs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) mfmsr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) xori r10,r10,(MSR_FE0|MSR_FE1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) mtmsrd r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) .Lreg=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) .rept 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) fmr .Lreg,.Lreg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) .Lreg=.Lreg+1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) FTR_SECTION_ELSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) * To denormalise we need to move a copy of the register to itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) * For POWER7 do that here for the first 32 VSX registers only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) mfmsr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) oris r10,r10,MSR_VSX@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) mtmsrd r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) .Lreg=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) .rept 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) XVCPSGNDP(.Lreg,.Lreg,.Lreg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) .Lreg=.Lreg+1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) b denorm_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) * To denormalise we need to move a copy of the register to itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) * For POWER8 we need to do that for all 64 VSX registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) .Lreg=32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) .rept 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) XVCPSGNDP(.Lreg,.Lreg,.Lreg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) .Lreg=.Lreg+1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) denorm_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) mfspr r11,SPRN_HSRR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) subi r11,r11,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) mtspr SPRN_HSRR0,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) mtcrf 0x80,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) ld r9,PACA_EXGEN+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) ld r10,PACA_EXGEN+EX_PPR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) mtspr SPRN_PPR,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) ld r10,PACA_EXGEN+EX_CFAR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) mtspr SPRN_CFAR,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) ld r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) ld r11,PACA_EXGEN+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) ld r12,PACA_EXGEN+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) ld r13,PACA_EXGEN+EX_R13(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) HRFI_TO_UNKNOWN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) b .
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) EXC_COMMON_BEGIN(denorm_exception_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) GEN_COMMON denorm_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) GEN_KVM denorm_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) #ifdef CONFIG_CBE_RAS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) INT_DEFINE_BEGIN(cbe_maintenance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) IVEC=0x1600
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) INT_DEFINE_END(cbe_maintenance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) GEN_INT_ENTRY cbe_maintenance, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) EXC_REAL_END(cbe_maintenance, 0x1600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) EXC_VIRT_NONE(0x5600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) EXC_COMMON_BEGIN(cbe_maintenance_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) GEN_COMMON cbe_maintenance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) bl cbe_maintenance_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) GEN_KVM cbe_maintenance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) #else /* CONFIG_CBE_RAS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) EXC_REAL_NONE(0x1600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) EXC_VIRT_NONE(0x5600, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) INT_DEFINE_BEGIN(altivec_assist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) IVEC=0x1700
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) INT_DEFINE_END(altivec_assist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) EXC_REAL_BEGIN(altivec_assist, 0x1700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) GEN_INT_ENTRY altivec_assist, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) EXC_REAL_END(altivec_assist, 0x1700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) GEN_INT_ENTRY altivec_assist, virt=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) EXC_VIRT_END(altivec_assist, 0x5700, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) EXC_COMMON_BEGIN(altivec_assist_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) GEN_COMMON altivec_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) #ifdef CONFIG_ALTIVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) bl altivec_assist_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) REST_NVGPRS(r1) /* instruction emulation may change GPRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) bl unknown_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) GEN_KVM altivec_assist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) #ifdef CONFIG_CBE_RAS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) INT_DEFINE_BEGIN(cbe_thermal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) IVEC=0x1800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) IHSRR=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) IKVM_SKIP=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) IKVM_REAL=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) INT_DEFINE_END(cbe_thermal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) GEN_INT_ENTRY cbe_thermal, virt=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) EXC_REAL_END(cbe_thermal, 0x1800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) EXC_VIRT_NONE(0x5800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) EXC_COMMON_BEGIN(cbe_thermal_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) GEN_COMMON cbe_thermal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) bl cbe_thermal_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) GEN_KVM cbe_thermal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) #else /* CONFIG_CBE_RAS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) EXC_REAL_NONE(0x1800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) EXC_VIRT_NONE(0x5800, 0x100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) #ifdef CONFIG_PPC_WATCHDOG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) INT_DEFINE_BEGIN(soft_nmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) IVEC=0x900
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) ISTACK=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) INT_DEFINE_END(soft_nmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) * Branch to soft_nmi_interrupt using the emergency stack. The emergency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) * stack is one that is usable by maskable interrupts so long as MSR_EE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) * remains off. It is used for recovery when something has corrupted the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) * normal kernel stack, for example. The "soft NMI" must not use the process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) * stack because we want irq disabled sections to avoid touching the stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) * at all (other than PMU interrupts), so use the emergency stack for this,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) * and run it entirely with interrupts hard disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) EXC_COMMON_BEGIN(soft_nmi_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) mfspr r11,SPRN_SRR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) mr r10,r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) ld r1,PACAEMERGSP(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) subi r1,r1,INT_FRAME_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) __GEN_COMMON_BODY soft_nmi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) * system_reset_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) li r10,IRQS_ALL_DISABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) lbz r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) std r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) ori r10,r10,PACA_IRQ_HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) bl soft_nmi_interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) /* Clear MSR_RI before setting SRR0 and SRR1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) li r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) mtmsrd r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) * Restore soft mask settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) ld r10,RESULT(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) ld r10,SOFTE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) stb r10,PACAIRQSOFTMASK(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) kuap_restore_amr r9, r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) EXCEPTION_RESTORE_REGS hsrr=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) #endif /* CONFIG_PPC_WATCHDOG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) * - If it was a decrementer interrupt, we bump the dec to max and and return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) * - If it was a doorbell we return immediately since doorbells are edge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) * triggered and won't automatically refire.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * - If it was a HMI we return immediately since we handled it in realmode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * and it won't refire.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) * This is called with r10 containing the value to OR to the paca field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) .macro MASKED_INTERRUPT hsrr=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) .if \hsrr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) masked_Hinterrupt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) masked_interrupt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) lbz r11,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) or r11,r11,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) stb r11,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) cmpwi r10,PACA_IRQ_DEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) bne 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) lis r10,0x7fff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) ori r10,r10,0xffff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) mtspr SPRN_DEC,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) #ifdef CONFIG_PPC_WATCHDOG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) b soft_nmi_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) b 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) 1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) beq 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) xori r12,r12,MSR_EE /* clear MSR_EE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) .if \hsrr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) mtspr SPRN_HSRR1,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) mtspr SPRN_SRR1,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) ori r11,r11,PACA_IRQ_HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) stb r11,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) 2: /* done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) ld r10,PACA_EXGEN+EX_CTR(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) mtctr r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) mtcrf 0x80,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) std r1,PACAR1(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) ld r9,PACA_EXGEN+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) ld r10,PACA_EXGEN+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) ld r11,PACA_EXGEN+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) ld r12,PACA_EXGEN+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) ld r13,PACA_EXGEN+EX_R13(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) /* May return to masked low address where r13 is not set up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) .if \hsrr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) HRFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) b .
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) TRAMP_REAL_BEGIN(stf_barrier_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) std r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) std r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) ld r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) ld r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) ori 31,31,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) .rept 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) b 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) /* Clobbers r10, r11, ctr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) .macro L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) ld r11,PACA_L1D_FLUSH_SIZE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) mtctr r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) /* order ld/st prior to dcbt stop all streams with flushing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) * The load addresses are at staggered offsets within cachelines,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) * which suits some pipelines better (on others it should not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) * hurt).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) ld r11,(0x80 + 8)*0(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) ld r11,(0x80 + 8)*1(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) ld r11,(0x80 + 8)*2(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) ld r11,(0x80 + 8)*3(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) ld r11,(0x80 + 8)*4(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) ld r11,(0x80 + 8)*5(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) ld r11,(0x80 + 8)*6(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) ld r11,(0x80 + 8)*7(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) addi r10,r10,0x80*8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) bdnz 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) TRAMP_REAL_BEGIN(entry_flush_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) std r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) std r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) std r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) mfctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) mtctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) ld r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) ld r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) ld r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) * The SCV entry flush happens with interrupts enabled, so it must disable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) * to prevent EXRFI being clobbered by NMIs (e.g., soft_nmi_common). r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) * (containing LR) does not need to be preserved here because scv entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) * puts 0 in the pt_regs, CTR can be clobbered for the same reason.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) TRAMP_REAL_BEGIN(scv_entry_flush_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) li r10,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) lbz r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) ori r10,r10,PACA_IRQ_HARD_DIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) stb r10,PACAIRQHAPPENED(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) std r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) ld r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) li r10,MSR_RI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) mtmsrd r10,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) TRAMP_REAL_BEGIN(rfi_flush_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) SET_SCRATCH0(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) GET_PACA(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) std r1,PACA_EXRFI+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) ld r1,PACAKSAVE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) std r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) std r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) std r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) mfctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) mtctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) ld r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) ld r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) ld r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) ld r1,PACA_EXRFI+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) GET_SCRATCH0(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) rfid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) TRAMP_REAL_BEGIN(hrfi_flush_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) SET_SCRATCH0(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) GET_PACA(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) std r1,PACA_EXRFI+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) ld r1,PACAKSAVE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) std r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) std r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) std r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) mfctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) mtctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) ld r9,PACA_EXRFI+EX_R9(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) ld r10,PACA_EXRFI+EX_R10(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) ld r11,PACA_EXRFI+EX_R11(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) ld r1,PACA_EXRFI+EX_R12(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) GET_SCRATCH0(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) hrfid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) TRAMP_REAL_BEGIN(rfscv_flush_fallback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) /* system call volatile */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) mr r7,r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) GET_PACA(r13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) mr r8,r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) ld r1,PACAKSAVE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) mfctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) ld r11,PACA_L1D_FLUSH_SIZE(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) mtctr r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) /* order ld/st prior to dcbt stop all streams with flushing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) * The load adresses are at staggered offsets within cachelines,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) * which suits some pipelines better (on others it should not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) * hurt).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) ld r11,(0x80 + 8)*0(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) ld r11,(0x80 + 8)*1(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) ld r11,(0x80 + 8)*2(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) ld r11,(0x80 + 8)*3(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) ld r11,(0x80 + 8)*4(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) ld r11,(0x80 + 8)*5(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) ld r11,(0x80 + 8)*6(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) ld r11,(0x80 + 8)*7(r10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) addi r10,r10,0x80*8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) bdnz 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) mtctr r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) li r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) li r10,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) li r11,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) mr r1,r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) mr r13,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) RFSCV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) USE_TEXT_SECTION()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) _GLOBAL(do_uaccess_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) UACCESS_FLUSH_FIXUP_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) L1D_DISPLACEMENT_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) _ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) EXPORT_SYMBOL(do_uaccess_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) MASKED_INTERRUPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) MASKED_INTERRUPT hsrr=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) kvmppc_skip_interrupt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) * Here all GPRs are unchanged from when the interrupt happened
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) * except for r13, which is saved in SPRG_SCRATCH0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) mfspr r13, SPRN_SRR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) addi r13, r13, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) mtspr SPRN_SRR0, r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) GET_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) b .
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) kvmppc_skip_Hinterrupt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) * Here all GPRs are unchanged from when the interrupt happened
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) * except for r13, which is saved in SPRG_SCRATCH0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) mfspr r13, SPRN_HSRR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) addi r13, r13, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) mtspr SPRN_HSRR0, r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) GET_SCRATCH0(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) HRFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) b .
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) * Relocation-on interrupts: A subset of the interrupts can be delivered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) * it. Addresses are the same as the original interrupt addresses, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) * offset by 0xc000000000004000.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) * It's impossible to receive interrupts below 0x300 via this mechanism.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) * KVM: None of these traps are from the guest ; anything that escalated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) * to HV=1 from HV=0 is delivered via real mode handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) * This uses the standard macro, since the original 0x300 vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) * only has extra guff for STAB-based processors -- which never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) * come here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) b __ppc64_runlatch_on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) USE_FIXED_SECTION(virt_trampolines)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) * All code below __end_interrupts is treated as soft-masked. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) * any code runs here with MSR[EE]=1, it must then cope with pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) * soft interrupt being raised (i.e., by ensuring it is replayed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) * The __end_interrupts marker must be past the out-of-line (OOL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) * handlers, so that they are copied to real address 0x100 when running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) * a relocatable kernel. This ensures they can be reached from the short
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) * directly, without using LOAD_HANDLER().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) .align 7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) .globl __end_interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) __end_interrupts:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) DEFINE_FIXED_SYMBOL(__end_interrupts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) #ifdef CONFIG_PPC_970_NAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) * Called by exception entry code if _TLF_NAPPING was set, this clears
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) * the NAPPING flag, and redirects the exception exit to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) * power4_fixup_nap_return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) .globl power4_fixup_nap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) EXC_COMMON_BEGIN(power4_fixup_nap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) andc r9,r9,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) std r9,TI_LOCAL_FLAGS(r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) LOAD_REG_ADDR(r10, power4_idle_nap_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) std r10,_NIP(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) power4_idle_nap_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) CLOSE_FIXED_SECTION(real_vectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) CLOSE_FIXED_SECTION(real_trampolines);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) CLOSE_FIXED_SECTION(virt_vectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) CLOSE_FIXED_SECTION(virt_trampolines);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) USE_TEXT_SECTION()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) /* MSR[RI] should be clear because this uses SRR[01] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) enable_machine_check:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) mflr r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) bcl 20,31,$+4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) 0: mflr r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) addi r3,r3,(1f - 0b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) mtspr SPRN_SRR0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) mfmsr r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) ori r3,r3,MSR_ME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) mtspr SPRN_SRR1,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) 1: mtlr r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) /* MSR[RI] should be clear because this uses SRR[01] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) disable_machine_check:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) mflr r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) bcl 20,31,$+4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) 0: mflr r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) addi r3,r3,(1f - 0b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) mtspr SPRN_SRR0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) mfmsr r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) li r4,MSR_ME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) andc r3,r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) mtspr SPRN_SRR1,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) RFI_TO_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) 1: mtlr r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) * Hash table stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) .balign IFETCH_ALIGN_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) do_hash_page:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) ori r0,r0,DSISR_BAD_FAULT_64S@l
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) and. r0,r5,r0 /* weird error? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) bne- handle_page_fault /* if not, try to insert a HPTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) * don't call hash_page, just fail the fault. This is required to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) * prevent re-entrancy problems in the hash code, namely perf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) * interrupts hitting while something holds H_PAGE_BUSY, and taking a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * hash fault. See the comment in hash_preload().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) ld r11, PACA_THREAD_INFO(r13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) lwz r0,TI_PREEMPT(r11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) andis. r0,r0,NMI_MASK@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) bne 77f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) * r3 contains the trap number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) * r4 contains the faulting address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) * r5 contains dsisr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) * r6 msr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) * at return r3 = 0 for success, 1 for page fault, negative for error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) bl __hash_page /* build HPTE if possible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) cmpdi r3,0 /* see if __hash_page succeeded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) /* Success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) beq interrupt_return /* Return from exception on success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) /* Error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) blt- 13f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) /* Reload DAR/DSISR into r4/r5 for the DABR check below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) ld r5,_DSISR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) #endif /* CONFIG_PPC_BOOK3S_64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) /* Here we have a page fault that hash_page can't handle. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) handle_page_fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) 11: andis. r0,r5,DSISR_DABRMATCH@h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) bne- handle_dabr_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) bl do_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) cmpdi r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) beq+ interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) mr r5,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) bl bad_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) /* We have a data breakpoint exception - handle it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) handle_dabr_fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) ld r5,_DSISR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) bl do_break
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) * do_break() may have changed the NV GPRS while handling a breakpoint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) * If so, we need to restore them with their updated values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) REST_NVGPRS(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) /* We have a page fault that hash_page could handle but HV refused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) * the PTE insertion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) 13: mr r5,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) ld r4,_DAR(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) bl low_hash_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) b interrupt_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) * We come here as a result of a DSI at a point where we don't want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) * to call hash_page, such as when we are accessing memory (possibly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) * user memory) inside a PMU interrupt that occurred while interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) * were soft-disabled. We want to invoke the exception handler for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) * the access, or panic if there isn't a handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) 77: addi r3,r1,STACK_FRAME_OVERHEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) li r5,SIGSEGV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) bl bad_page_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) b interrupt_return
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags