^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/mem_encrypt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/realmode.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/crash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/sev-es.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) struct real_mode_header *real_mode_header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) u32 *trampoline_cr4_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) /* Hold the pgd entry used on booting additional CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) pgd_t trampoline_pgd_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) void load_trampoline_pgtable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) load_cr3(initial_page_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * This function is called before exiting to real-mode and that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * fail with CR4.PCIDE still set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) if (boot_cpu_has(X86_FEATURE_PCID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) cr4_clear_bits(X86_CR4_PCIDE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) write_cr3(real_mode_header->trampoline_pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * The CR3 write above will not flush global TLB entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * Stale, global entries from previous page tables may still be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * present. Flush those stale entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * This ensures that memory accessed while running with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * trampoline_pgd is *actually* mapped into trampoline_pgd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) __flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) void __init reserve_real_mode(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) phys_addr_t mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) size_t size = real_mode_size_needed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (!size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) WARN_ON(slab_is_available());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* Has to be under 1M so we can execute real-mode AP code. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) mem = memblock_find_in_range(0, 1<<20, size, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (!mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) pr_info("No sub-1M memory is available for the trampoline\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) memblock_reserve(mem, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) set_real_mode_mem(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) crash_reserve_low_1M();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static void sme_sev_setup_real_mode(struct trampoline_header *th)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #ifdef CONFIG_AMD_MEM_ENCRYPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (sme_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) th->flags |= TH_FLAGS_SME_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (sev_es_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Skip the call to verify_cpu() in secondary_startup_64 as it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * will cause #VC exceptions when the AP can't handle them yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) th->start = (u64) secondary_startup_64_no_verify;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (sev_es_setup_ap_jump_table(real_mode_header))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) panic("Failed to get/update SEV-ES AP Jump Table");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static void __init setup_real_mode(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) u16 real_mode_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) const u32 *rel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) u32 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) unsigned char *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned long phys_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct trampoline_header *trampoline_header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) u64 *trampoline_pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) u64 efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) base = (unsigned char *)real_mode_header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * If SME is active, the trampoline area will need to be in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * decrypted memory in order to bring up other processors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * successfully. This is not needed for SEV.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (sme_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) set_memory_decrypted((unsigned long)base, size >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) memcpy(base, real_mode_blob, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) phys_base = __pa(base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) real_mode_seg = phys_base >> 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) rel = (u32 *) real_mode_relocs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /* 16-bit segment relocations. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) count = *rel++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) while (count--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u16 *seg = (u16 *) (base + *rel++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) *seg = real_mode_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) /* 32-bit linear relocations. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) count = *rel++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) while (count--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) u32 *ptr = (u32 *) (base + *rel++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) *ptr += phys_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* Must be perfomed *after* relocation. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) trampoline_header = (struct trampoline_header *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) __va(real_mode_header->trampoline_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) trampoline_header->start = __pa_symbol(startup_32_smp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) trampoline_header->gdt_limit = __BOOT_DS + 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) trampoline_header->gdt_base = __pa_symbol(boot_gdt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * Some AMD processors will #GP(0) if EFER.LMA is set in WRMSR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * so we need to mask it out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) rdmsrl(MSR_EFER, efer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) trampoline_header->efer = efer & ~EFER_LMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) trampoline_header->start = (u64) secondary_startup_64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) trampoline_cr4_features = &trampoline_header->cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) *trampoline_cr4_features = mmu_cr4_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) trampoline_header->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /* Map the real mode stub as virtual == physical */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) trampoline_pgd[0] = trampoline_pgd_entry.pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Include the entirety of the kernel mapping into the trampoline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * PGD. This way, all mappings present in the normal kernel page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * tables are usable while running on trampoline_pgd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) for (i = pgd_index(__PAGE_OFFSET); i < PTRS_PER_PGD; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) trampoline_pgd[i] = init_top_pgt[i].pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) sme_sev_setup_real_mode(trampoline_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * reserve_real_mode() gets called very early, to guarantee the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * availability of low memory. This is before the proper kernel page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * tables are set up, so we cannot set page permissions in that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * function. Also trampoline code will be executed by APs so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * need to mark it executable at do_pre_smp_initcalls() at least,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * thus run it as a early_initcall().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static void __init set_real_mode_permissions(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) unsigned char *base = (unsigned char *) real_mode_header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) size_t size = PAGE_ALIGN(real_mode_blob_end - real_mode_blob);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) size_t ro_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) PAGE_ALIGN(real_mode_header->ro_end) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) __pa(base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) size_t text_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) PAGE_ALIGN(real_mode_header->ro_end) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) real_mode_header->text_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned long text_start =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) (unsigned long) __va(real_mode_header->text_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) set_memory_nx((unsigned long) base, size >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) set_memory_ro((unsigned long) base, ro_size >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) set_memory_x((unsigned long) text_start, text_size >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static int __init init_real_mode(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (!real_mode_header)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) panic("Real mode trampoline was not allocated");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) setup_real_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) set_real_mode_permissions();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) early_initcall(init_real_mode);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags