^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 1991,1992 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * entry_32.S contains the system-call and low-level fault and trap handling routines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Stack layout while running C code:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * ptrace needs to have all registers on the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * If the order here is changed, it needs to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * updated in fork.c:copy_process(), signal.c:do_signal(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * ptrace.c and ptrace.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * 0(%esp) - %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * 4(%esp) - %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * 8(%esp) - %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * C(%esp) - %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * 10(%esp) - %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * 14(%esp) - %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * 18(%esp) - %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * 1C(%esp) - %ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * 20(%esp) - %es
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * 24(%esp) - %fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * 2C(%esp) - orig_eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * 30(%esp) - %eip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * 34(%esp) - %cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * 38(%esp) - %eflags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * 3C(%esp) - %oldesp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * 40(%esp) - %oldss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/irqflags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/segment.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/processor-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/irq_vectors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/cpufeatures.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/alternative-asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/smap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <asm/frame.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <asm/trapnr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <asm/nospec-branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include "calling.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) .section .entry.text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define PTI_SWITCH_MASK (1 << PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * User gs save/restore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * %gs is used for userland TLS and kernel only uses it for stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * canary which is required to be at %gs:20 by gcc. Read the comment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * at the top of stackprotector.h for more info.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * Local labels 98 and 99 are used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #ifdef CONFIG_X86_32_LAZY_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* unfortunately push/pop can't be no-op */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) .macro PUSH_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) pushl $0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) .macro POP_GS pop=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) addl $(4 + \pop), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) .macro POP_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* all the rest are no-op */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) .macro PTGS_TO_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .macro PTGS_TO_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) .macro GS_TO_REG reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) .macro REG_TO_PTGS reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) .macro SET_KERNEL_GS reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #else /* CONFIG_X86_32_LAZY_GS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .macro PUSH_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pushl %gs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .macro POP_GS pop=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) 98: popl %gs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) .if \pop <> 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) add $\pop, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) .macro POP_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .pushsection .fixup, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 99: movl $0, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) jmp 98b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) _ASM_EXTABLE(98b, 99b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) .macro PTGS_TO_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 98: mov PT_GS(%esp), %gs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) .macro PTGS_TO_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) .pushsection .fixup, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 99: movl $0, PT_GS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) jmp 98b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) _ASM_EXTABLE(98b, 99b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .macro GS_TO_REG reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) movl %gs, \reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) .macro REG_TO_PTGS reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) movl \reg, PT_GS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) .macro SET_KERNEL_GS reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) movl $(__KERNEL_STACK_CANARY), \reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) movl \reg, %gs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #endif /* CONFIG_X86_32_LAZY_GS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* Unconditionally switch to user cr3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) .macro SWITCH_TO_USER_CR3 scratch_reg:req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) movl %cr3, \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) orl $PTI_SWITCH_MASK, \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) movl \scratch_reg, %cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .macro BUG_IF_WRONG_CR3 no_user_check=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #ifdef CONFIG_DEBUG_ENTRY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .if \no_user_check == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /* coming from usermode? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) testl $USER_SEGMENT_RPL_MASK, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) jz .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* On user-cr3? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) movl %cr3, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) testl $PTI_SWITCH_MASK, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) jnz .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* From userspace with kernel cr3 - BUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) ud2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * Switch to kernel cr3 if not already loaded and return current cr3 in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) movl %cr3, \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* Test if we are already on kernel CR3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) testl $PTI_SWITCH_MASK, \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) jz .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) andl $(~PTI_SWITCH_MASK), \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) movl \scratch_reg, %cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* Return original CR3 in \scratch_reg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) orl $PTI_SWITCH_MASK, \scratch_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define CS_FROM_ENTRY_STACK (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #define CS_FROM_USER_CR3 (1 << 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #define CS_FROM_KERNEL (1 << 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #define CS_FROM_ESPFIX (1 << 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) .macro FIXUP_FRAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * The high bits of the CS dword (__csh) are used for CS_FROM_*.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * Clear them in case hardware didn't do this for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) andl $0x0000ffff, 4*4(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #ifdef CONFIG_VM86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) testl $X86_EFLAGS_VM, 5*4(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) jnz .Lfrom_usermode_no_fixup_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) testl $USER_SEGMENT_RPL_MASK, 4*4(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) jnz .Lfrom_usermode_no_fixup_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) orl $CS_FROM_KERNEL, 4*4(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * When we're here from kernel mode; the (exception) stack looks like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * 6*4(%esp) - <previous context>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * 5*4(%esp) - flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * 4*4(%esp) - cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * 3*4(%esp) - ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * 2*4(%esp) - orig_eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * 1*4(%esp) - gs / function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * 0*4(%esp) - fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * Lets build a 5 entry IRET frame after that, such that struct pt_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * is complete and in particular regs->sp is correct. This gives us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * the original 6 enties as gap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * 14*4(%esp) - <previous context>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * 13*4(%esp) - gap / flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * 12*4(%esp) - gap / cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * 11*4(%esp) - gap / ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * 10*4(%esp) - gap / orig_eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * 9*4(%esp) - gap / gs / function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * 8*4(%esp) - gap / fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * 7*4(%esp) - ss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * 6*4(%esp) - sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * 5*4(%esp) - flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * 4*4(%esp) - cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * 3*4(%esp) - ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * 2*4(%esp) - orig_eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * 1*4(%esp) - gs / function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * 0*4(%esp) - fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) pushl %ss # ss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) pushl %esp # sp (points at ss)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) addl $7*4, (%esp) # point sp back at the previous context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) pushl 7*4(%esp) # flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) pushl 7*4(%esp) # cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) pushl 7*4(%esp) # ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) pushl 7*4(%esp) # orig_eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) pushl 7*4(%esp) # gs / function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) pushl 7*4(%esp) # fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) .Lfrom_usermode_no_fixup_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) .macro IRET_FRAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * We're called with %ds, %es, %fs, and %gs from the interrupted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * frame, so we shouldn't use them. Also, we may be in ESPFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * mode and therefore have a nonzero SS base and an offset ESP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * so any attempt to access the stack needs to use SS. (except for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * accesses through %esp, which automatically use SS.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) testl $CS_FROM_KERNEL, 1*4(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) jz .Lfinished_frame_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * Reconstruct the 3 entry IRET frame right after the (modified)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * regs->sp without lowering %esp in between, such that an NMI in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * middle doesn't scribble our stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) pushl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) movl 5*4(%esp), %eax # (modified) regs->sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) movl 4*4(%esp), %ecx # flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) movl %ecx, %ss:-1*4(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) movl 3*4(%esp), %ecx # cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) andl $0x0000ffff, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) movl %ecx, %ss:-2*4(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) movl 2*4(%esp), %ecx # ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) movl %ecx, %ss:-3*4(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) movl 1*4(%esp), %ecx # eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) movl %ecx, %ss:-4*4(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) popl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) lea -4*4(%eax), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) .Lfinished_frame_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) .macro SAVE_ALL pt_regs_ax=%eax switch_stacks=0 skip_gs=0 unwind_espfix=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) .if \skip_gs == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) PUSH_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) pushl %fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) movl $(__KERNEL_PERCPU), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) movl %eax, %fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) .if \unwind_espfix > 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) UNWIND_ESPFIX_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) FIXUP_FRAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) pushl %es
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) pushl %ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) pushl \pt_regs_ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) pushl %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) pushl %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) pushl %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) pushl %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) pushl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) pushl %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) movl $(__USER_DS), %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) movl %edx, %ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) movl %edx, %es
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) .if \skip_gs == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) SET_KERNEL_GS %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /* Switch to kernel stack if necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) .if \switch_stacks > 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) SWITCH_TO_KERNEL_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .macro SAVE_ALL_NMI cr3_reg:req unwind_espfix=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) SAVE_ALL unwind_espfix=\unwind_espfix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) BUG_IF_WRONG_CR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * Now switch the CR3 when PTI is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * We can enter with either user or kernel cr3, the code will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * store the old cr3 in \cr3_reg and switches to the kernel cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) SWITCH_TO_KERNEL_CR3 scratch_reg=\cr3_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) .macro RESTORE_INT_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) popl %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) popl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) popl %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) popl %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) popl %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) popl %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) .macro RESTORE_REGS pop=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) RESTORE_INT_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 1: popl %ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 2: popl %es
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 3: popl %fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) POP_GS \pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) IRET_FRAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) .pushsection .fixup, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 4: movl $0, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) jmp 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 5: movl $0, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) jmp 2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 6: movl $0, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) jmp 3b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) _ASM_EXTABLE(1b, 4b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) _ASM_EXTABLE(2b, 5b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) _ASM_EXTABLE(3b, 6b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) POP_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) .macro RESTORE_ALL_NMI cr3_reg:req pop=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Now switch the CR3 when PTI is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * We enter with kernel cr3 and switch the cr3 to the value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * stored on \cr3_reg, which is either a user or a kernel cr3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ALTERNATIVE "jmp .Lswitched_\@", "", X86_FEATURE_PTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) testl $PTI_SWITCH_MASK, \cr3_reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) jz .Lswitched_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) /* User cr3 in \cr3_reg - write it to hardware cr3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) movl \cr3_reg, %cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) .Lswitched_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) BUG_IF_WRONG_CR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) RESTORE_REGS pop=\pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) .macro CHECK_AND_APPLY_ESPFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) ALTERNATIVE "jmp .Lend_\@", "", X86_BUG_ESPFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * are returning to the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * See comments in process.c:copy_thread() for details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) movb PT_OLDSS(%esp), %ah
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) movb PT_CS(%esp), %al
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) jne .Lend_\@ # returning to user-space with LDT SS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * Setup and switch to ESPFIX stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * We're returning to userspace with a 16 bit stack. The CPU will not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * restore the high word of ESP for us on executing iret... This is an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * "official" bug of all the x86-compatible CPUs, which we can work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * around to make dosemu and wine happy. We do this by preloading the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * high word of ESP with the high word of the userspace ESP while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * compensating for the offset by changing to the ESPFIX segment with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * a base address that matches for the difference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) mov %esp, %edx /* load kernel esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) mov PT_OLDESP(%esp), %eax /* load userspace esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) mov %dx, %ax /* eax: new kernel esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) sub %eax, %edx /* offset (low word is 0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) shr $16, %edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) pushl $__ESPFIX_SS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) pushl %eax /* new kernel esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * Disable interrupts, but do not irqtrace this section: we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * will soon execute iret and the tracer was already set to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * the irqstate after the IRET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) DISABLE_INTERRUPTS(CLBR_ANY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) lss (%esp), %esp /* switch to espfix segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) #endif /* CONFIG_X86_ESPFIX32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * Called with pt_regs fully populated and kernel segments loaded,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * so we can access PER_CPU and use the integer registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * We need to be very careful here with the %esp switch, because an NMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * can happen everywhere. If the NMI handler finds itself on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * entry-stack, it will overwrite the task-stack and everything we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * copied there. So allocate the stack-frame on the task-stack and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * switch to it before we do any copying.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .macro SWITCH_TO_KERNEL_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) BUG_IF_WRONG_CR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * %eax now contains the entry cr3 and we carry it forward in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * that register for the time this macro runs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /* Are we on the entry stack? Bail out if not! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) movl PER_CPU_VAR(cpu_entry_area), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) subl %esp, %ecx /* ecx = (end of entry_stack) - esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) cmpl $SIZEOF_entry_stack, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) jae .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /* Load stack pointer into %esi and %edi */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) movl %esp, %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) movl %esi, %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* Move %edi to the top of the entry stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) andl $(MASK_entry_stack), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) addl $(SIZEOF_entry_stack), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /* Load top of task-stack into %edi */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) movl TSS_entry2task_stack(%edi), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /* Special case - entry from kernel mode via entry stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) #ifdef CONFIG_VM86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) movl PT_EFLAGS(%esp), %ecx # mix EFLAGS and CS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) movb PT_CS(%esp), %cl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) movl PT_CS(%esp), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) andl $SEGMENT_RPL_MASK, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) cmpl $USER_RPL, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) jb .Lentry_from_kernel_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /* Bytes to copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) movl $PTREGS_SIZE, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) #ifdef CONFIG_VM86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) testl $X86_EFLAGS_VM, PT_EFLAGS(%esi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) jz .Lcopy_pt_regs_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Stack-frame contains 4 additional segment registers when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * coming from VM86 mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) addl $(4 * 4), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) .Lcopy_pt_regs_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* Allocate frame on task-stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) subl %ecx, %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /* Switch to task-stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) movl %edi, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * We are now on the task-stack and can safely copy over the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * stack-frame
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) shrl $2, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) rep movsl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) jmp .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) .Lentry_from_kernel_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * This handles the case when we enter the kernel from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * kernel-mode and %esp points to the entry-stack. When this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * happens we need to switch to the task-stack to run C code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * but switch back to the entry-stack again when we approach
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * iret and return to the interrupted code-path. This usually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * happens when we hit an exception while restoring user-space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * segment registers on the way back to user-space or when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * sysenter handler runs with eflags.tf set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * When we switch to the task-stack here, we can't trust the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * contents of the entry-stack anymore, as the exception handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * might be scheduled out or moved to another CPU. Therefore we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * copy the complete entry-stack to the task-stack and set a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * marker in the iret-frame (bit 31 of the CS dword) to detect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * what we've done on the iret path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * On the iret path we copy everything back and switch to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * entry-stack, so that the interrupted kernel code-path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * continues on the same stack it was interrupted with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * Be aware that an NMI can happen anytime in this code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * %esi: Entry-Stack pointer (same as %esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * %edi: Top of the task stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * %eax: CR3 on kernel entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Calculate number of bytes on the entry stack in %ecx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) movl %esi, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* %ecx to the top of entry-stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) andl $(MASK_entry_stack), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) addl $(SIZEOF_entry_stack), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /* Number of bytes on the entry stack to %ecx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) sub %esi, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* Mark stackframe as coming from entry stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) orl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * Test the cr3 used to enter the kernel and add a marker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * so that we can switch back to it before iret.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) testl $PTI_SWITCH_MASK, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) jz .Lcopy_pt_regs_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) orl $CS_FROM_USER_CR3, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * %esi and %edi are unchanged, %ecx contains the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * bytes to copy. The code at .Lcopy_pt_regs_\@ will allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * the stack-frame on task-stack and copy everything over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) jmp .Lcopy_pt_regs_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * Switch back from the kernel stack to the entry stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * The %esp register must point to pt_regs on the task stack. It will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * first calculate the size of the stack-frame to copy, depending on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * whether we return to VM86 mode or not. With that it uses 'rep movsl'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * to copy the contents of the stack over to the entry stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * We must be very careful here, as we can't trust the contents of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * task-stack once we switched to the entry-stack. When an NMI happens
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * while on the entry-stack, the NMI handler will switch back to the top
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * of the task stack, overwriting our stack-frame we are about to copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * Therefore we switch the stack only after everything is copied over.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) .macro SWITCH_TO_ENTRY_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) /* Bytes to copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) movl $PTREGS_SIZE, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) #ifdef CONFIG_VM86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) testl $(X86_EFLAGS_VM), PT_EFLAGS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) jz .Lcopy_pt_regs_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /* Additional 4 registers to copy when returning to VM86 mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) addl $(4 * 4), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) .Lcopy_pt_regs_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /* Initialize source and destination for movsl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) subl %ecx, %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) movl %esp, %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /* Save future stack pointer in %ebx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) movl %edi, %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /* Copy over the stack-frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) shrl $2, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) rep movsl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * Switch to entry-stack - needs to happen after everything is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * copied because the NMI handler will overwrite the task-stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * when on entry-stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) movl %ebx, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * This macro handles the case when we return to kernel-mode on the iret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * path and have to switch back to the entry stack and/or user-cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * See the comments below the .Lentry_from_kernel_\@ label in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * SWITCH_TO_KERNEL_STACK macro for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) .macro PARANOID_EXIT_TO_KERNEL_MODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * Test if we entered the kernel with the entry-stack. Most
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * likely we did not, because this code only runs on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * return-to-kernel path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) testl $CS_FROM_ENTRY_STACK, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) jz .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /* Unlikely slow-path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /* Clear marker from stack-frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) andl $(~CS_FROM_ENTRY_STACK), PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* Copy the remaining task-stack contents to entry-stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) movl %esp, %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /* Bytes on the task-stack to ecx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) movl PER_CPU_VAR(cpu_tss_rw + TSS_sp1), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) subl %esi, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /* Allocate stack-frame on entry-stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) subl %ecx, %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * Save future stack-pointer, we must not switch until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * copy is done, otherwise the NMI handler could destroy the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * contents of the task-stack we are about to copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) movl %edi, %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /* Do the copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) shrl $2, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) rep movsl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) /* Safe to switch to entry-stack now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) movl %ebx, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * We came from entry-stack and need to check if we also need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * switch back to user cr3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) testl $CS_FROM_USER_CR3, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) jz .Lend_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) /* Clear marker from stack-frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) andl $(~CS_FROM_USER_CR3), PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) SWITCH_TO_USER_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) .Lend_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * idtentry - Macro to generate entry stubs for simple IDT entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * @vector: Vector number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * @asmsym: ASM symbol for the entry point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * @cfunc: C function to be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * @has_error_code: Hardware pushed error code on stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) .macro idtentry vector asmsym cfunc has_error_code:req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) SYM_CODE_START(\asmsym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) .if \has_error_code == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) pushl $0 /* Clear the error code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* Push the C-function address into the GS slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) pushl $\cfunc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /* Invoke the common exception entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) jmp handle_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) SYM_CODE_END(\asmsym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) .macro idtentry_irq vector cfunc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) .p2align CONFIG_X86_L1_CACHE_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) SYM_CODE_START_LOCAL(asm_\cfunc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) SAVE_ALL switch_stacks=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) ENCODE_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) movl %esp, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) movl PT_ORIG_EAX(%esp), %edx /* get the vector from stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) movl $-1, PT_ORIG_EAX(%esp) /* no syscall to restart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) call \cfunc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) jmp handle_exception_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) SYM_CODE_END(asm_\cfunc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) .macro idtentry_sysvec vector cfunc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) idtentry \vector asm_\cfunc \cfunc has_error_code=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * Include the defines which emit the idt entries which are shared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * shared between 32 and 64 bit and emit the __irqentry_text_* markers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * so the stacktrace boundary checks work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) .globl __irqentry_text_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) __irqentry_text_start:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) #include <asm/idtentry.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) .globl __irqentry_text_end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) __irqentry_text_end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * %eax: prev task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * %edx: next task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) .pushsection .text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) SYM_CODE_START(__switch_to_asm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * Save callee-saved registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * This must match the order in struct inactive_task_frame
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) pushl %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) pushl %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) pushl %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) pushl %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * Flags are saved to prevent AC leakage. This could go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * away if objtool would have 32bit support to verify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) * the STAC/CLAC correctness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) pushfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) /* switch stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) movl %esp, TASK_threadsp(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) movl TASK_threadsp(%edx), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) #ifdef CONFIG_STACKPROTECTOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) movl TASK_stack_canary(%edx), %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) #ifdef CONFIG_RETPOLINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * When switching from a shallower to a deeper call stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * the RSB may either underflow or use entries populated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * with userspace addresses. On CPUs where those concerns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * exist, overwrite the RSB with entries which capture
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * speculative execution to prevent attack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /* Restore flags or the incoming task to restore AC state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) popfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /* restore callee-saved registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) popl %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) popl %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) popl %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) popl %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) jmp __switch_to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) SYM_CODE_END(__switch_to_asm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * The unwinder expects the last frame on the stack to always be at the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * offset from the end of the page, which allows it to validate the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * Calling schedule_tail() directly would break that convention because its an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * asmlinkage function so its argument has to be pushed on the stack. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * wrapper creates a proper "end of stack" frame header before the call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) .pushsection .text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) SYM_FUNC_START(schedule_tail_wrapper)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) call schedule_tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) SYM_FUNC_END(schedule_tail_wrapper)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * A newly forked process directly context switches into this address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * eax: prev task we switched from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * ebx: kernel thread func (NULL for user thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * edi: kernel thread arg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) .pushsection .text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) SYM_CODE_START(ret_from_fork)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) call schedule_tail_wrapper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) testl %ebx, %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) jnz 1f /* kernel threads are uncommon */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) /* When we fork, we trace the syscall return in the child, too. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) movl %esp, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) call syscall_exit_to_user_mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) jmp .Lsyscall_32_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) /* kernel thread */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) 1: movl %edi, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) CALL_NOSPEC ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) * A kernel thread is allowed to return here after successfully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * calling kernel_execve(). Exit to userspace to complete the execve()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * syscall.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) movl $0, PT_EAX(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) jmp 2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) SYM_CODE_END(ret_from_fork)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * All code from here through __end_SYSENTER_singlestep_region is subject
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * to being single-stepped if a user program sets TF and executes SYSENTER.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * There is absolutely nothing that we can do to prevent this from happening
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * (thanks Intel!). To keep our handling of this situation as simple as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * possible, we handle TF just like AC and NT, except that our #DB handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * will ignore all of the single-step traps generated in this range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * 32-bit SYSENTER entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * if X86_FEATURE_SEP is available. This is the preferred system call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * entry on 32-bit systems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * The SYSENTER instruction, in principle, should *only* occur in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * vDSO. In practice, a small number of Android devices were shipped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * with a copy of Bionic that inlined a SYSENTER instruction. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * never happened in any of Google's Bionic versions -- it only happened
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * in a narrow range of Intel-provided versions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * SYSENTER does not save anything on the stack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * and does not save old EIP (!!!), ESP, or EFLAGS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * user and/or vm86 state), we explicitly disable the SYSENTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * instruction in vm86 mode by reprogramming the MSRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * Arguments:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * eax system call number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * ebx arg1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * ecx arg2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * edx arg3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * esi arg4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * edi arg5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * ebp user stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * 0(%ebp) arg6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) SYM_FUNC_START(entry_SYSENTER_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * On entry-stack with all userspace-regs live - save and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * restore eflags and %eax to use it as scratch-reg for the cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * switch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) pushfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) BUG_IF_WRONG_CR3 no_user_check=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) SWITCH_TO_KERNEL_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) popfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) /* Stack empty again, switch to task stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) movl TSS_entry2task_stack(%esp), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) .Lsysenter_past_esp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) pushl $__USER_DS /* pt_regs->ss */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) pushl $0 /* pt_regs->sp (placeholder) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) pushfl /* pt_regs->flags (except IF = 0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) pushl $__USER_CS /* pt_regs->cs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) pushl $0 /* pt_regs->ip = 0 (placeholder) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) pushl %eax /* pt_regs->orig_ax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest, stack already switched */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * SYSENTER doesn't filter flags, so we need to clear NT, AC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * and TF ourselves. To save a few cycles, we can check whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * either was set instead of doing an unconditional popfq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * This needs to happen before enabling interrupts so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * we don't get preempted with NT set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * If TF is set, we will single-step all the way to here -- do_debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * will ignore all the traps. (Yes, this is slow, but so is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * single-stepping in general. This allows us to avoid having
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * a more complicated code to handle the case where a user program
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * forces us to single-step through the SYSENTER entry code.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * NB.: .Lsysenter_fix_flags is a label with the code under it moved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * out-of-line as an optimization: NT is unlikely to be set in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * majority of the cases and instead of polluting the I$ unnecessarily,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * we're keeping that code behind a branch which will predict as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * not-taken and therefore its instructions won't be fetched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) jnz .Lsysenter_fix_flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) .Lsysenter_flags_fixed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) movl %esp, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) call do_SYSENTER_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) testl %eax, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) jz .Lsyscall_32_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) STACKLEAK_ERASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /* Opportunistic SYSEXIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * Setup entry stack - we keep the pointer in %eax and do the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * switch after almost all user-state is restored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) /* Load entry stack pointer and allocate frame for eflags/eax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) movl PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) subl $(2*4), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /* Copy eflags and eax to entry stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) movl PT_EFLAGS(%esp), %edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) movl PT_EAX(%esp), %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) movl %edi, (%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) movl %esi, 4(%eax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) /* Restore user registers and segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) movl PT_EIP(%esp), %edx /* pt_regs->ip */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) 1: mov PT_FS(%esp), %fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) PTGS_TO_GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) popl %ebx /* pt_regs->bx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) popl %esi /* pt_regs->si */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) popl %edi /* pt_regs->di */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) popl %ebp /* pt_regs->bp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /* Switch to entry stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) movl %eax, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) /* Now ready to switch the cr3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) SWITCH_TO_USER_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * Restore all flags except IF. (We restore IF separately because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * STI gives a one-instruction window in which we won't be interrupted,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) * whereas POPF does not.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) btrl $X86_EFLAGS_IF_BIT, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) BUG_IF_WRONG_CR3 no_user_check=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) popfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * Return back to the vDSO, which will pop ecx and edx.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * Don't bother with DS and ES (they already contain __USER_DS).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) sti
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) sysexit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) .pushsection .fixup, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) 2: movl $0, PT_FS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) jmp 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) .popsection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) _ASM_EXTABLE(1b, 2b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) PTGS_TO_GS_EX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) .Lsysenter_fix_flags:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) pushl $X86_EFLAGS_FIXED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) popfl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) jmp .Lsysenter_flags_fixed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) SYM_ENTRY(__end_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) SYM_FUNC_END(entry_SYSENTER_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * 32-bit legacy system call entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * 32-bit x86 Linux system calls traditionally used the INT $0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) * instruction. INT $0x80 lands here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * This entry point can be used by any 32-bit perform system calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * Instances of INT $0x80 can be found inline in various programs and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * libraries. It is also used by the vDSO's __kernel_vsyscall
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * fallback for hardware that doesn't support a faster entry method.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * Restarted 32-bit system calls also fall back to INT $0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * regardless of what instruction was originally used to do the system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * call. (64-bit programs can use INT $0x80 as well, but they can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * only run on 64-bit kernels and therefore land in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * entry_INT80_compat.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * This is considered a slow path. It is not used by most libc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * implementations on modern hardware except during process startup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * Arguments:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * eax system call number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * ebx arg1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * ecx arg2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) * edx arg3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * esi arg4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) * edi arg5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) * ebp arg6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) SYM_FUNC_START(entry_INT80_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) pushl %eax /* pt_regs->orig_ax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) movl %esp, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) call do_int80_syscall_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) .Lsyscall_32_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) STACKLEAK_ERASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) restore_all_switch_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) SWITCH_TO_ENTRY_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) CHECK_AND_APPLY_ESPFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) /* Switch back to user CR3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) SWITCH_TO_USER_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) BUG_IF_WRONG_CR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /* Restore user state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) RESTORE_REGS pop=4 # skip orig_eax/error_code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) .Lirq_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * when returning from IPI handler and when returning from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) * scheduler to user-space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) INTERRUPT_RETURN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) .section .fixup, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) SYM_CODE_START(asm_iret_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) pushl $0 # no error code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) pushl $iret_error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) #ifdef CONFIG_DEBUG_ENTRY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * The stack-frame here is the one that iret faulted on, so its a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * return-to-user frame. We are on kernel-cr3 because we come here from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * the fixup code. This confuses the CR3 checker, so switch to user-cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * as the checker expects it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) SWITCH_TO_USER_CR3 scratch_reg=%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) jmp handle_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) SYM_CODE_END(asm_iret_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) _ASM_EXTABLE(.Lirq_return, asm_iret_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) SYM_FUNC_END(entry_INT80_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) .macro FIXUP_ESPFIX_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) * Switch back for ESPFIX stack to the normal zerobased stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * We can't call C functions using the ESPFIX stack. This code reads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) * the high word of the segment base from the GDT and swiches to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) * normal stack and adjusts ESP with the matching offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) * We might be on user CR3 here, so percpu data is not mapped and we can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) * access the GDT through the percpu segment. Instead, use SGDT to find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) * the cpu_entry_area alias of the GDT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) /* fixup the stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) pushl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) subl $2*4, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) sgdt (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) movl 2(%esp), %ecx /* GDT address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * Careful: ECX is a linear pointer, so we need to force base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * zero. %cs is the only known-linear segment we have right now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) mov %cs:GDT_ESPFIX_OFFSET + 4(%ecx), %al /* bits 16..23 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) mov %cs:GDT_ESPFIX_OFFSET + 7(%ecx), %ah /* bits 24..31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) shl $16, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) addl $2*4, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) popl %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) addl %esp, %eax /* the adjusted stack pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) pushl $__KERNEL_DS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) lss (%esp), %esp /* switch to the normal stack segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) .macro UNWIND_ESPFIX_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) /* It's safe to clobber %eax, all other regs need to be preserved */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) movl %ss, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) /* see if on espfix stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) cmpw $__ESPFIX_SS, %ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) jne .Lno_fixup_\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) /* switch to normal stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) FIXUP_ESPFIX_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) .Lno_fixup_\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) SYM_CODE_START_LOCAL_NOALIGN(handle_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) /* the function address is in %gs's slot on the stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) ENCODE_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) /* fixup %gs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) GS_TO_REG %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) movl PT_GS(%esp), %edi # get the function address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) REG_TO_PTGS %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) SET_KERNEL_GS %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) /* fixup orig %eax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) movl PT_ORIG_EAX(%esp), %edx # get the error code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) movl %esp, %eax # pt_regs pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) CALL_NOSPEC edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) handle_exception_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) #ifdef CONFIG_VM86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) movb PT_CS(%esp), %al
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) * We can be coming here from child spawned by kernel_thread().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) movl PT_CS(%esp), %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) andl $SEGMENT_RPL_MASK, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) cmpl $USER_RPL, %eax # returning to v8086 or userspace ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) jnb ret_to_user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) PARANOID_EXIT_TO_KERNEL_MODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) BUG_IF_WRONG_CR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) RESTORE_REGS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) jmp .Lirq_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) ret_to_user:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) movl %esp, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) jmp restore_all_switch_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) SYM_CODE_END(handle_exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) SYM_CODE_START(asm_exc_double_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * This is a task gate handler, not an interrupt gate handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * The error code is on the stack, but the stack is otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * empty. Interrupts are off. Our state is sane with the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * exceptions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * - CR0.TS is set. "TS" literally means "task switched".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * - EFLAGS.NT is set because we're a "nested task".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * - The doublefault TSS has back_link set and has been marked busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * - TR points to the doublefault TSS and the normal TSS is busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) * - CR3 is the normal kernel PGD. This would be delightful, except
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * that the CPU didn't bother to save the old CR3 anywhere. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * would make it very awkward to return back to the context we came
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * The rest of EFLAGS is sanitized for us, so we don't need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) * worry about AC or DF.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * Don't even bother popping the error code. It's always zero,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * and ignoring it makes us a bit more robust against buggy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * hypervisor task gate implementations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * We will manually undo the task switch instead of doing a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * task-switching IRET.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) clts /* clear CR0.TS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) pushl $X86_EFLAGS_FIXED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) popfl /* clear EFLAGS.NT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) call doublefault_shim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) /* We don't support returning, so we have no IRET here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) hlt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) jmp 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) SYM_CODE_END(asm_exc_double_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * NMI is doubly nasty. It can happen on the first instruction of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) * switched stacks. We handle both conditions by simply checking whether we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * interrupted kernel code running on the SYSENTER stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) SYM_CODE_START(asm_exc_nmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * ESPFIX_SS is only ever set on the return to user path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) * after we've switched to the entry stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) pushl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) movl %ss, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) cmpw $__ESPFIX_SS, %ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) popl %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) je .Lnmi_espfix_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) pushl %eax # pt_regs->orig_ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) SAVE_ALL_NMI cr3_reg=%edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) ENCODE_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) xorl %edx, %edx # zero error code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) movl %esp, %eax # pt_regs pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /* Are we currently on the SYSENTER stack? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) movl PER_CPU_VAR(cpu_entry_area), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) cmpl $SIZEOF_entry_stack, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) jb .Lnmi_from_sysenter_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) /* Not on SYSENTER stack. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) call exc_nmi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) jmp .Lnmi_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) .Lnmi_from_sysenter_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) * We're on the SYSENTER stack. Switch off. No one (not even debug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) * is using the thread stack right now, so it's safe for us to use it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) movl %esp, %ebx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) call exc_nmi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) movl %ebx, %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) .Lnmi_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) testl $CS_FROM_ESPFIX, PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) jnz .Lnmi_from_espfix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) CHECK_AND_APPLY_ESPFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) RESTORE_ALL_NMI cr3_reg=%edi pop=4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) jmp .Lirq_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) #ifdef CONFIG_X86_ESPFIX32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) .Lnmi_espfix_stack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * Create the pointer to LSS back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) pushl %ss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) pushl %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) addl $4, (%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) /* Copy the (short) IRET frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) pushl 4*4(%esp) # flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) pushl 4*4(%esp) # cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) pushl 4*4(%esp) # ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) pushl %eax # orig_ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) SAVE_ALL_NMI cr3_reg=%edi unwind_espfix=1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) ENCODE_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) /* clear CS_FROM_KERNEL, set CS_FROM_ESPFIX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) xorl $(CS_FROM_ESPFIX | CS_FROM_KERNEL), PT_CS(%esp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) xorl %edx, %edx # zero error code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) movl %esp, %eax # pt_regs pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) jmp .Lnmi_from_sysenter_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) .Lnmi_from_espfix:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) RESTORE_ALL_NMI cr3_reg=%edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) * Because we cleared CS_FROM_KERNEL, IRET_FRAME 'forgot' to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) * fix up the gap and long frame:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) * 3 - original frame (exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) * 2 - ESPFIX block (above)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) * 6 - gap (FIXUP_FRAME)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) * 5 - long frame (FIXUP_FRAME)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) * 1 - orig_ax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) lss (1+5+6)*4(%esp), %esp # back to espfix stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) jmp .Lirq_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) SYM_CODE_END(asm_exc_nmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) .pushsection .text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) SYM_CODE_START(rewind_stack_do_exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) /* Prevent any naive code from trying to unwind to our caller. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) xorl %ebp, %ebp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) movl PER_CPU_VAR(cpu_current_top_of_stack), %esi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) call do_exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) 1: jmp 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) SYM_CODE_END(rewind_stack_do_exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) .popsection
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags