Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * AMD Memory Encryption Support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2016 Advanced Micro Devices, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Author: Tom Lendacky <thomas.lendacky@amd.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #define DISABLE_BRANCH_PROFILING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/dma-direct.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/swiotlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/mem_encrypt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <linux/cc_platform.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <asm/setup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <asm/bootparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <asm/processor-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <asm/msr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #include <asm/cmdline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #include "mm_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * Since SME related variables are set early in the boot process they must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  * reside in the .data section so as not to be zeroed out when the .bss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * section is later cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) u64 sme_me_mask __section(".data") = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) u64 sev_status __section(".data") = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) u64 sev_check_data __section(".data") = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) EXPORT_SYMBOL(sme_me_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) DEFINE_STATIC_KEY_FALSE(sev_enable_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) EXPORT_SYMBOL_GPL(sev_enable_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) bool sev_enabled __section(".data");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) /* Buffer used for early in-place encryption by BSP, no locking needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static char sme_early_buffer[PAGE_SIZE] __initdata __aligned(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * This routine does not change the underlying encryption setting of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  * page(s) that map this memory. It assumes that eventually the memory is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  * meant to be accessed as either encrypted or decrypted but the contents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57)  * are currently not in the desired state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * This routine follows the steps outlined in the AMD64 Architecture
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) static void __init __sme_early_enc_dec(resource_size_t paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 				       unsigned long size, bool enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	void *src, *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	if (!sme_me_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	wbinvd();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	 * There are limited number of early mapping slots, so map (at most)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	 * one page at time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	while (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 		len = min_t(size_t, sizeof(sme_early_buffer), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		 * Create mappings for the current and desired format of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		 * the memory. Use a write-protected mapping for the source.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 		src = enc ? early_memremap_decrypted_wp(paddr, len) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 			    early_memremap_encrypted_wp(paddr, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 		dst = enc ? early_memremap_encrypted(paddr, len) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 			    early_memremap_decrypted(paddr, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		 * If a mapping can't be obtained to perform the operation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		 * then eventual access of that area in the desired mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 		 * will cause a crash.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		BUG_ON(!src || !dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		 * Use a temporary buffer, of cache-line multiple size, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		 * avoid data corruption as documented in the APM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		memcpy(sme_early_buffer, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		memcpy(dst, sme_early_buffer, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		early_memunmap(dst, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		early_memunmap(src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		paddr += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		size -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) void __init sme_early_encrypt(resource_size_t paddr, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	__sme_early_enc_dec(paddr, size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void __init sme_early_decrypt(resource_size_t paddr, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	__sme_early_enc_dec(paddr, size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static void __init __sme_early_map_unmap_mem(void *vaddr, unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 					     bool map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	unsigned long paddr = (unsigned long)vaddr - __PAGE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	pmdval_t pmd_flags, pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	/* Use early_pmd_flags but remove the encryption mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	pmd_flags = __sme_clr(early_pmd_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		pmd = map ? (paddr & PMD_MASK) + pmd_flags : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		__early_make_pgtable((unsigned long)vaddr, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		vaddr += PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		paddr += PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		size = (size <= PMD_SIZE) ? 0 : size - PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	} while (size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	flush_tlb_local();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) void __init sme_unmap_bootdata(char *real_mode_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	struct boot_params *boot_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	unsigned long cmdline_paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	if (!sme_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	/* Get the command line address before unmapping the real_mode_data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	boot_data = (struct boot_params *)real_mode_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	if (!cmdline_paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) void __init sme_map_bootdata(char *real_mode_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	struct boot_params *boot_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	unsigned long cmdline_paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	if (!sme_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	__sme_early_map_unmap_mem(real_mode_data, sizeof(boot_params), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	/* Get the command line address after mapping the real_mode_data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	boot_data = (struct boot_params *)real_mode_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	cmdline_paddr = boot_data->hdr.cmd_line_ptr | ((u64)boot_data->ext_cmd_line_ptr << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	if (!cmdline_paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	__sme_early_map_unmap_mem(__va(cmdline_paddr), COMMAND_LINE_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) void __init sme_early_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	if (!sme_me_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	early_pmd_flags = __sme_set(early_pmd_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	__supported_pte_mask = __sme_set(__supported_pte_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	/* Update the protection map with memory encryption mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	for (i = 0; i < ARRAY_SIZE(protection_map); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		protection_map[i] = pgprot_encrypted(protection_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	if (sev_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 		swiotlb_force = SWIOTLB_FORCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	pgprot_t old_prot, new_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	unsigned long pfn, pa, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	pte_t new_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	switch (level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	case PG_LEVEL_4K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		pfn = pte_pfn(*kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 		old_prot = pte_pgprot(*kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	case PG_LEVEL_2M:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		pfn = pmd_pfn(*(pmd_t *)kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		old_prot = pmd_pgprot(*(pmd_t *)kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	case PG_LEVEL_1G:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		pfn = pud_pfn(*(pud_t *)kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		old_prot = pud_pgprot(*(pud_t *)kpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	new_prot = old_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	if (enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		pgprot_val(new_prot) |= _PAGE_ENC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		pgprot_val(new_prot) &= ~_PAGE_ENC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	/* If prot is same then do nothing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	if (pgprot_val(old_prot) == pgprot_val(new_prot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	pa = pfn << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	size = page_level_size(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	 * We are going to perform in-place en-/decryption and change the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	 * physical page attribute from C=1 to C=0 or vice versa. Flush the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	 * caches to ensure that data gets accessed with the correct C-bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	clflush_cache_range(__va(pa), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	/* Encrypt/decrypt the contents in-place */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	if (enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		sme_early_encrypt(pa, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 		sme_early_decrypt(pa, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	/* Change the page encryption mask. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	new_pte = pfn_pte(pfn, new_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	set_pte_atomic(kpte, new_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static int __init early_set_memory_enc_dec(unsigned long vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 					   unsigned long size, bool enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	unsigned long vaddr_end, vaddr_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	unsigned long psize, pmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	int split_page_size_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	int level, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	pte_t *kpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	vaddr_next = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	vaddr_end = vaddr + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	for (; vaddr < vaddr_end; vaddr = vaddr_next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		kpte = lookup_address(vaddr, &level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		if (!kpte || pte_none(*kpte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 			ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		if (level == PG_LEVEL_4K) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 			__set_clr_pte_enc(kpte, level, enc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 			vaddr_next = (vaddr & PAGE_MASK) + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 		psize = page_level_size(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 		pmask = page_level_mask(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 		 * Check whether we can change the large page in one go.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 		 * We request a split when the address is not aligned and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		 * the number of pages to set/clear encryption bit is smaller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 		 * than the number of pages in the large page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		if (vaddr == (vaddr & pmask) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 		    ((vaddr_end - vaddr) >= psize)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 			__set_clr_pte_enc(kpte, level, enc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 			vaddr_next = (vaddr & pmask) + psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 		 * The virtual address is part of a larger page, create the next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		 * level page table mapping (4K or 2M). If it is part of a 2M
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		 * page then we request a split of the large page into 4K
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		 * chunks. A 1GB large page is split into 2M pages, resp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 		if (level == PG_LEVEL_2M)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 			split_page_size_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 			split_page_size_mask = 1 << PG_LEVEL_2M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 		 * kernel_physical_mapping_change() does not flush the TLBs, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 		 * a TLB flush is required after we exit from the for loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 		kernel_physical_mapping_change(__pa(vaddr & pmask),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 					       __pa((vaddr_end & pmask) + psize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 					       split_page_size_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	__flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	return early_set_memory_enc_dec(vaddr, size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	return early_set_memory_enc_dec(vaddr, size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)  * SME and SEV are very similar but they are not the same, so there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)  * times that the kernel will need to distinguish between SME and SEV. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)  * sme_active() and sev_active() functions are used for this.  When a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)  * distinction isn't needed, the mem_encrypt_active() function can be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)  * The trampoline code is a good example for this requirement.  Before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)  * paging is activated, SME will access all memory as decrypted, but SEV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)  * will access all memory as encrypted.  So, when APs are being brought
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)  * up under SME the trampoline area cannot be encrypted, whereas under SEV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)  * the trampoline area must be encrypted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) bool sme_active(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	return sme_me_mask && !sev_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) bool sev_active(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	return sev_status & MSR_AMD64_SEV_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) EXPORT_SYMBOL_GPL(sev_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /* Needs to be called from non-instrumentable code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) bool noinstr sev_es_active(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	return sev_status & MSR_AMD64_SEV_ES_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) bool force_dma_unencrypted(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	 * For SEV, all DMA must be to unencrypted addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	if (sev_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 		return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	 * For SME, all DMA must be to unencrypted addresses if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	 * device does not support DMA to addresses that include the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	 * encryption mask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	if (sme_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 		u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 		u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 						dev->bus_dma_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 		if (dma_dev_mask <= dma_enc_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 			return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) void __init mem_encrypt_free_decrypted_mem(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	unsigned long vaddr, vaddr_end, npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	vaddr = (unsigned long)__start_bss_decrypted_unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	vaddr_end = (unsigned long)__end_bss_decrypted;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	npages = (vaddr_end - vaddr) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	 * The unused memory range was mapped decrypted, change the encryption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 	 * attribute from decrypted to encrypted before freeing it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 	if (mem_encrypt_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 		r = set_memory_encrypted(vaddr, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 		if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 			pr_warn("failed to free unused decrypted pages\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	free_init_pages("unused decrypted", vaddr, vaddr_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static void print_mem_encrypt_feature_info(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	pr_info("AMD Memory Encryption Features active:");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	/* Secure Memory Encryption */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 	if (sme_active()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 		 * SME is mutually exclusive with any of the SEV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 		 * features below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 		pr_cont(" SME\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	/* Secure Encrypted Virtualization */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 	if (sev_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 		pr_cont(" SEV");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	/* Encrypted Register State */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 	if (sev_es_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 		pr_cont(" SEV-ES");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* Architecture __weak replacement functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) void __init mem_encrypt_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	if (!sme_me_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	/* Call into SWIOTLB to update the SWIOTLB DMA buffers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	swiotlb_update_mem_attributes();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	 * With SEV, we need to unroll the rep string I/O instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	if (sev_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 		static_branch_enable(&sev_enable_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	print_mem_encrypt_feature_info();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)