^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/arch/x86/kernel/head_64.S -- start in 32bit and switch to 64bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2000 Karsten Keil <kkeil@suse.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2001,2002 Andi Kleen <ak@suse.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/threads.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/segment.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/msr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/processor-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/nops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "../entry/calling.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/nospec-branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #ifdef CONFIG_PARAVIRT_XXL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/paravirt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define INTERRUPT_RETURN iretq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * We are not able to switch in one step to the final KERNEL ADDRESS SPACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * because we need identity-mapped pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define l4_index(x) (((x) >> 39) & 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) L4_PAGE_OFFSET = l4_index(__PAGE_OFFSET_BASE_L4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) L4_START_KERNEL = l4_index(__START_KERNEL_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) L3_START_KERNEL = pud_index(__START_KERNEL_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) .text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) __HEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) .code64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) SYM_CODE_START_NOALIGN(startup_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * and someone has loaded an identity mapped page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * for us. These identity mapped page tables map all of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * kernel pages and possibly all of memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * %rsi holds a physical pointer to real_mode_data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * We come here either directly from a 64bit bootloader, or from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * arch/x86/boot/compressed/head_64.S.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * We only come here initially at boot nothing else comes here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Since we may be loaded at an address different from what we were
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * compiled to run at we first fixup the physical addresses in our page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * tables and then reload them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* Set up the stack for verify_cpu(), similar to initial_stack below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) leaq _text(%rip), %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) pushq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) call startup_64_setup_env
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) popq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* Now switch to __KERNEL_CS so IRET works reliably */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pushq $__KERNEL_CS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) leaq .Lon_kernel_cs(%rip), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) pushq %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) lretq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) .Lon_kernel_cs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* Sanitize CPU configuration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) call verify_cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Perform pagetable fixups. Additionally, if SME is active, encrypt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * the kernel and retrieve the modifier (SME encryption mask if SME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * is active) to be added to the initial pgdir entry that will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * programmed into CR3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) leaq _text(%rip), %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) pushq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) call __startup_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) popq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) /* Form the CR3 value being sure to include the CR3 modifier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) addq $(early_top_pgt - __START_KERNEL_map), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) jmp 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) SYM_CODE_END(startup_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) SYM_CODE_START(secondary_startup_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * At this point the CPU runs in 64bit mode CS.L = 1 CS.D = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * and someone has loaded a mapped page table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * %rsi holds a physical pointer to real_mode_data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * We come here either from startup_64 (using physical addresses)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * or from trampoline.S (using virtual addresses).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * Using virtual addresses from trampoline.S removes the need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * to have any identity mapped pages in the kernel page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * after the boot processor executes this code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Sanitize CPU configuration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) call verify_cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * The secondary_startup_64_no_verify entry point is only used by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * SEV-ES guests. In those guests the call to verify_cpu() would cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * #VC exceptions which can not be handled at this stage of secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * CPU bringup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * All non SEV-ES systems, especially Intel systems, need to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * verify_cpu() above to make sure NX is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * Retrieve the modifier (SME encryption mask if SME is active) to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * added to the initial pgdir entry that will be programmed into CR3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pushq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) call __startup_secondary_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) popq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /* Form the CR3 value being sure to include the CR3 modifier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) addq $(init_top_pgt - __START_KERNEL_map), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* Enable PAE mode, PGE and LA57 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #ifdef CONFIG_X86_5LEVEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) testl $1, __pgtable_l5_enabled(%rip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) jz 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) orl $X86_CR4_LA57, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) movq %rcx, %cr4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /* Setup early boot stage 4-/5-level pagetables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) addq phys_base(%rip), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * For SEV guests: Verify that the C-bit is correct. A malicious
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * hypervisor could lie about the C-bit position to perform a ROP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * attack on the guest by writing to the unencrypted stack and wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * the next RET instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * %rsi carries pointer to realmode data and is callee-clobbered. Save
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * and restore it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) pushq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) movq %rax, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) call sev_verify_cbit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) popq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* Switch to new page-table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) movq %rax, %cr3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /* Ensure I am executing from virtual addresses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) movq $1f, %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) ANNOTATE_RETPOLINE_SAFE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) jmp *%rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * We must switch to a new descriptor in kernel space for the GDT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * because soon the kernel won't have access anymore to the userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * addresses where we're currently running on. We have to do that here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * because in 32bit we couldn't load a 64bit linear address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) lgdt early_gdt_descr(%rip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /* set up data segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) xorl %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) movl %eax,%ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) movl %eax,%ss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) movl %eax,%es
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * We don't really need to load %fs or %gs, but load them anyway
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * to kill any stale realmode selectors. This allows execution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * under VT hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) movl %eax,%fs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) movl %eax,%gs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* Set up %gs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * The base of %gs always points to fixed_percpu_data. If the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * stack protector canary is enabled, it is located at %gs:40.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * Note that, on SMP, the boot cpu uses init data section until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * the per cpu areas are set up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) movl $MSR_GS_BASE,%ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) movl initial_gs(%rip),%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) movl initial_gs+4(%rip),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) wrmsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * Setup a boot time stack - Any secondary CPU will have lost its stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * by now because the cr3-switch above unmaps the real-mode stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) movq initial_stack(%rip), %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) /* Setup and Load IDT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) pushq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) call early_setup_idt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) popq %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* Check if nx is implemented */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) movl $0x80000001, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) cpuid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) movl %edx,%edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* Setup EFER (Extended Feature Enable Register) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) movl $MSR_EFER, %ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) rdmsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) btsl $_EFER_SCE, %eax /* Enable System Call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) btl $20,%edi /* No Execute supported? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) jnc 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) btsl $_EFER_NX, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) btsq $_PAGE_BIT_NX,early_pmd_flags(%rip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 1: wrmsr /* Make changes effective */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /* Setup cr0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) movl $CR0_STATE, %eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* Make changes effective */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) movq %rax, %cr0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* zero EFLAGS after setting rsp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) pushq $0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) popfq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* rsi is pointer to real mode structure with interesting info.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pass it to C */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) movq %rsi, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) .Ljump_to_C_code:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * Jump to run C code and to be on a real kernel address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * Since we are running on identity-mapped space we have to jump
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * to the full 64bit address, this is only possible as indirect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * jump. In addition we need to ensure %cs is set so we make this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * a far return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * Note: do not change to far jump indirect with 64bit offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * AMD does not support far jump indirect with 64bit offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * AMD64 Architecture Programmer's Manual, Volume 3: states only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * JMP FAR mem16:16 FF /5 Far jump indirect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * with the target specified by a far pointer in memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * JMP FAR mem16:32 FF /5 Far jump indirect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * with the target specified by a far pointer in memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * Intel64 does support 64bit offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * Software Developer Manual Vol 2: states:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * FF /5 JMP m16:16 Jump far, absolute indirect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * address given in m16:16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * FF /5 JMP m16:32 Jump far, absolute indirect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * address given in m16:32.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * REX.W + FF /5 JMP m16:64 Jump far, absolute indirect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * address given in m16:64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) pushq $.Lafter_lret # put return address on stack for unwinder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) xorl %ebp, %ebp # clear frame pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) movq initial_code(%rip), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) pushq $__KERNEL_CS # set correct cs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) pushq %rax # target address in negative space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) lretq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) .Lafter_lret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) SYM_CODE_END(secondary_startup_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #include "verify_cpu.S"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) #include "sev_verify_cbit.S"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * up already except stack. We just set up stack here. Then call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * start_secondary() via .Ljump_to_C_code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) SYM_CODE_START(start_cpu0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) movq initial_stack(%rip), %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) jmp .Ljump_to_C_code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) SYM_CODE_END(start_cpu0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) #ifdef CONFIG_AMD_MEM_ENCRYPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * VC Exception handler used during early boot when running on kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * addresses, but before the switch to the idt_table can be made.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * The early_idt_handler_array can't be used here because it calls into a lot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * of __init code and this handler is also used during CPU offlining/onlining.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * Therefore this handler ends up in the .text section so that it stays around
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * when .init.text is freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) SYM_CODE_START_NOALIGN(vc_boot_ghcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) UNWIND_HINT_IRET_REGS offset=8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /* Build pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) PUSH_AND_CLEAR_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /* Call C handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) movq %rsp, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) movq ORIG_RAX(%rsp), %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) movq initial_vc_handler(%rip), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) ANNOTATE_RETPOLINE_SAFE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) call *%rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /* Unwind pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) POP_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /* Remove Error Code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) addq $8, %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* Pure iret required here - don't use INTERRUPT_RETURN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) iretq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) SYM_CODE_END(vc_boot_ghcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /* Both SMP bootup and ACPI suspend change these variables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) __REFDATA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) .balign 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) SYM_DATA(initial_code, .quad x86_64_start_kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) SYM_DATA(initial_gs, .quad INIT_PER_CPU_VAR(fixed_percpu_data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) #ifdef CONFIG_AMD_MEM_ENCRYPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) SYM_DATA(initial_vc_handler, .quad handle_vc_boot_ghcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * The SIZEOF_PTREGS gap is a convention which helps the in-kernel unwinder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * reliably detect the end of the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - SIZEOF_PTREGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) __FINITDATA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) __INIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) SYM_CODE_START(early_idt_handler_array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) i = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) .rept NUM_EXCEPTION_VECTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) UNWIND_HINT_IRET_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pushq $0 # Dummy error code, to make stack frame uniform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) UNWIND_HINT_IRET_REGS offset=8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) pushq $i # 72(%rsp) Vector number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) jmp early_idt_handler_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) UNWIND_HINT_IRET_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) i = i + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) UNWIND_HINT_IRET_REGS offset=16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) SYM_CODE_END(early_idt_handler_array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) SYM_CODE_START_LOCAL(early_idt_handler_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * The stack is the hardware frame, an error code or zero, and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * vector number.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) cld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) incl early_recursion_flag(%rip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* The vector number is currently in the pt_regs->di slot. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) pushq %rsi /* pt_regs->si */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) movq 8(%rsp), %rsi /* RSI = vector number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) movq %rdi, 8(%rsp) /* pt_regs->di = RDI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) pushq %rdx /* pt_regs->dx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) pushq %rcx /* pt_regs->cx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) pushq %rax /* pt_regs->ax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) pushq %r8 /* pt_regs->r8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) pushq %r9 /* pt_regs->r9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) pushq %r10 /* pt_regs->r10 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) pushq %r11 /* pt_regs->r11 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) pushq %rbx /* pt_regs->bx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) pushq %rbp /* pt_regs->bp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pushq %r12 /* pt_regs->r12 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) pushq %r13 /* pt_regs->r13 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) pushq %r14 /* pt_regs->r14 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) pushq %r15 /* pt_regs->r15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) UNWIND_HINT_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) movq %rsp,%rdi /* RDI = pt_regs; RSI is already trapnr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) call do_early_exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) decl early_recursion_flag(%rip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) jmp restore_regs_and_return_to_kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) SYM_CODE_END(early_idt_handler_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) #ifdef CONFIG_AMD_MEM_ENCRYPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * VC Exception handler used during very early boot. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * early_idt_handler_array can't be used because it returns via the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * paravirtualized INTERRUPT_RETURN and pv-ops don't work that early.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * This handler will end up in the .init.text section and not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * available to boot secondary CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) SYM_CODE_START_NOALIGN(vc_no_ghcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) UNWIND_HINT_IRET_REGS offset=8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /* Build pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) PUSH_AND_CLEAR_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /* Call C handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) movq %rsp, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) movq ORIG_RAX(%rsp), %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) call do_vc_no_ghcb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /* Unwind pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) POP_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* Remove Error Code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) addq $8, %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /* Pure iret required here - don't use INTERRUPT_RETURN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) iretq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) SYM_CODE_END(vc_no_ghcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) #define SYM_DATA_START_PAGE_ALIGNED(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) #ifdef CONFIG_PAGE_TABLE_ISOLATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * Each PGD needs to be 8k long and 8k aligned. We do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * ever go out to userspace with these, so we do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * strictly *need* the second page, but this allows us to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * have a single set_pgd() implementation that does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * need to worry about whether it has 4k or 8k to work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * This ensures PGDs are 8k long:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #define PTI_USER_PGD_FILL 512
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /* This ensures they are 8k-aligned: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #define SYM_DATA_START_PTI_ALIGNED(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) SYM_START(name, SYM_L_GLOBAL, .balign 2 * PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) #define SYM_DATA_START_PTI_ALIGNED(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) SYM_DATA_START_PAGE_ALIGNED(name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) #define PTI_USER_PGD_FILL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) /* Automate the creation of 1 to 1 mapping pmd entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) #define PMDS(START, PERM, COUNT) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) i = 0 ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) .rept (COUNT) ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) .quad (START) + (i << PMD_SHIFT) + (PERM) ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) i = i + 1 ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) __INITDATA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) .balign 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) SYM_DATA_START_PTI_ALIGNED(early_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) .fill 512,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) .fill PTI_USER_PGD_FILL,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) SYM_DATA_END(early_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) SYM_DATA_START_PAGE_ALIGNED(early_dynamic_pgts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) SYM_DATA_END(early_dynamic_pgts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) SYM_DATA(early_recursion_flag, .long 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) .data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) #if defined(CONFIG_XEN_PV) || defined(CONFIG_PVH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) .org init_top_pgt + L4_PAGE_OFFSET*8, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) .org init_top_pgt + L4_START_KERNEL*8, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) .fill PTI_USER_PGD_FILL,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) SYM_DATA_END(init_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) SYM_DATA_START_PAGE_ALIGNED(level3_ident_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) .fill 511, 8, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) SYM_DATA_END(level3_ident_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) SYM_DATA_START_PAGE_ALIGNED(level2_ident_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * Since I easily can, map the first 1G.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * Don't set NX because code runs from these pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * Note: This sets _PAGE_GLOBAL despite whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * the CPU supports it or it is enabled. But,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * the CPU should ignore the bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) SYM_DATA_END(level2_ident_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) SYM_DATA_START_PTI_ALIGNED(init_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) .fill 512,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) .fill PTI_USER_PGD_FILL,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) SYM_DATA_END(init_top_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) #ifdef CONFIG_X86_5LEVEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) SYM_DATA_START_PAGE_ALIGNED(level4_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) .fill 511,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) SYM_DATA_END(level4_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) SYM_DATA_START_PAGE_ALIGNED(level3_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) .fill L3_START_KERNEL,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /* (2^48-(2*1024*1024*1024)-((2^39)*511))/(2^30) = 510 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) .quad level2_kernel_pgt - __START_KERNEL_map + _KERNPG_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) .quad level2_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) SYM_DATA_END(level3_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * 512 MB kernel mapping. We spend a full page on this pagetable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * The kernel code+data+bss must not be bigger than that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * (NOTE: at +512MB starts the module area, see MODULES_VADDR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * If you want to increase this then increase MODULES_VADDR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * too.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * This table is eventually used by the kernel during normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * runtime. Care must be taken to clear out undesired bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * later, like _PAGE_RW or _PAGE_GLOBAL in some cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) PMDS(0, __PAGE_KERNEL_LARGE_EXEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) KERNEL_IMAGE_SIZE/PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) SYM_DATA_END(level2_kernel_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) .fill (512 - 4 - FIXMAP_PMD_NUM),8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) pgtno = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) .rept (FIXMAP_PMD_NUM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) + _PAGE_TABLE_NOENC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) pgtno = pgtno + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /* 6 MB reserved space + a 2MB hole */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) .fill 4,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) SYM_DATA_END(level2_fixmap_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) SYM_DATA_START_PAGE_ALIGNED(level1_fixmap_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) .rept (FIXMAP_PMD_NUM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) .fill 512,8,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) .endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) SYM_DATA_END(level1_fixmap_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) #undef PMDS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) SYM_DATA(early_gdt_descr, .word GDT_ENTRIES*8-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) SYM_DATA_LOCAL(early_gdt_descr_base, .quad INIT_PER_CPU_VAR(gdt_page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* This must match the first entry in level2_kernel_pgt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) SYM_DATA(phys_base, .quad 0x0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) EXPORT_SYMBOL(phys_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) #include "../../x86/xen/xen-head.S"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) __PAGE_ALIGNED_BSS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) SYM_DATA_START_PAGE_ALIGNED(empty_zero_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) .skip PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) SYM_DATA_END(empty_zero_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) EXPORT_SYMBOL(empty_zero_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags