^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel-based Virtual Machine driver for Linux
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This module enables machines with Intel VT-x extensions to run virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * machines without emulation or binary translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2006 Qumranet, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright 2010 Red Hat, Inc. and/or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Avi Kivity <avi@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Yaniv Kamay <yaniv@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mod_devicetable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/objtool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sched/smt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/tboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/trace_events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/entry-kvm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/apic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/debugreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/desc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/fpu/internal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/idtentry.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/irq_remapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/kexec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/mce.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <asm/mshyperv.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <asm/mwait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <asm/spec-ctrl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <asm/virtext.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <asm/vmx.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "capabilities.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include "cpuid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include "evmcs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include "irq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include "kvm_cache_regs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include "lapic.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include "mmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "nested.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "pmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include "vmcs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include "vmcs12.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include "vmx.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) MODULE_AUTHOR("Qumranet");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #ifdef MODULE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static const struct x86_cpu_id vmx_cpu_id[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) bool __read_mostly enable_vpid = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) module_param_named(vpid, enable_vpid, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static bool __read_mostly enable_vnmi = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) bool __read_mostly flexpriority_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) bool __read_mostly enable_ept = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) module_param_named(ept, enable_ept, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) bool __read_mostly enable_unrestricted_guest = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) module_param_named(unrestricted_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) enable_unrestricted_guest, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) bool __read_mostly enable_ept_ad_bits = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static bool __read_mostly emulate_invalid_guest_state = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) module_param(emulate_invalid_guest_state, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static bool __read_mostly fasteoi = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) module_param(fasteoi, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) bool __read_mostly enable_apicv = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) module_param(enable_apicv, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * If nested=1, nested virtualization is supported, i.e., guests may use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * VMX and be a hypervisor for its own guests. If nested=0, guests may not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * use VMX instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) static bool __read_mostly nested = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) module_param(nested, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) bool __read_mostly enable_pml = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) module_param_named(pml, enable_pml, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static bool __read_mostly dump_invalid_vmcs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) module_param(dump_invalid_vmcs, bool, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define MSR_BITMAP_MODE_X2APIC 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define MSR_BITMAP_MODE_X2APIC_APICV 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #define KVM_VMX_TSC_MULTIPLIER_MAX 0xffffffffffffffffULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Guest_tsc -> host_tsc conversion requires 64-bit division. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static int __read_mostly cpu_preemption_timer_multi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static bool __read_mostly enable_preemption_timer = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) extern bool __read_mostly allow_smaller_maxphyaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define KVM_VM_CR0_ALWAYS_ON \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) RTIT_STATUS_BYTECNT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * List of MSRs that can be directly passed to the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * In addition to these x2apic and PT MSRs are handled specially.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) MSR_IA32_SPEC_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) MSR_IA32_PRED_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) MSR_IA32_TSC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) MSR_FS_BASE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) MSR_GS_BASE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) MSR_KERNEL_GS_BASE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) MSR_IA32_SYSENTER_CS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) MSR_IA32_SYSENTER_ESP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) MSR_IA32_SYSENTER_EIP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) MSR_CORE_C1_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) MSR_CORE_C3_RESIDENCY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) MSR_CORE_C6_RESIDENCY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) MSR_CORE_C7_RESIDENCY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * These 2 parameters are used to config the controls for Pause-Loop Exiting:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * ple_gap: upper bound on the amount of time between two successive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * executions of PAUSE in a loop. Also indicate if ple enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * According to test, this time is usually smaller than 128 cycles.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * ple_window: upper bound on the amount of time a guest is allowed to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * in a PAUSE loop. Tests indicate that most spinlocks are held for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * less than 2^12 cycles
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * Time is measured based on a counter that runs at the same rate as the TSC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * refer SDM volume 3b section 21.6.13 & 22.1.3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) module_param(ple_gap, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) module_param(ple_window, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /* Default doubles per-vcpu window every exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) module_param(ple_window_grow, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* Default resets per-vcpu window every exit to ple_window. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) module_param(ple_window_shrink, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* Default is to compute the maximum so we can never overflow. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static unsigned int ple_window_max = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) module_param(ple_window_max, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Default is SYSTEM mode, 1 for host-guest mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) int __read_mostly pt_mode = PT_MODE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) module_param(pt_mode, int, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static DEFINE_MUTEX(vmx_l1d_flush_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* Storage for pre module init parameter parsing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) const char *option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) bool for_parse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) } vmentry_l1d_param[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) [VMENTER_L1D_FLUSH_AUTO] = {"auto", true},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) [VMENTER_L1D_FLUSH_NEVER] = {"never", true},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) [VMENTER_L1D_FLUSH_COND] = {"cond", true},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) [VMENTER_L1D_FLUSH_ALWAYS] = {"always", true},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #define L1D_CACHE_ORDER 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static void *vmx_l1d_flush_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (!boot_cpu_has_bug(X86_BUG_L1TF)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (!enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) u64 msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /* If set to auto use the default l1tf mitigation method */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) switch (l1tf_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) case L1TF_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) l1tf = VMENTER_L1D_FLUSH_NEVER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) case L1TF_MITIGATION_FLUSH_NOWARN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) case L1TF_MITIGATION_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) case L1TF_MITIGATION_FLUSH_NOSMT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) l1tf = VMENTER_L1D_FLUSH_COND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) case L1TF_MITIGATION_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) case L1TF_MITIGATION_FULL_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) l1tf = VMENTER_L1D_FLUSH_ALWAYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) l1tf = VMENTER_L1D_FLUSH_ALWAYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * This allocation for vmx_l1d_flush_pages is not tied to a VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * lifetime and so should not be charged to a memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) vmx_l1d_flush_pages = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * Initialize each page with a different pattern in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * order to protect against KSM in the nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * virtualization case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) l1tf_vmx_mitigation = l1tf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (l1tf != VMENTER_L1D_FLUSH_NEVER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) static_branch_enable(&vmx_l1d_should_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static_branch_disable(&vmx_l1d_should_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (l1tf == VMENTER_L1D_FLUSH_COND)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static_branch_enable(&vmx_l1d_flush_cond);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static_branch_disable(&vmx_l1d_flush_cond);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) static int vmentry_l1d_flush_parse(const char *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (vmentry_l1d_param[i].for_parse &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) sysfs_streq(s, vmentry_l1d_param[i].option))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) int l1tf, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) l1tf = vmentry_l1d_flush_parse(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (l1tf < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return l1tf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!boot_cpu_has(X86_BUG_L1TF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Has vmx_init() run already? If not then this is the pre init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * parameter parsing. In that case just store the value and let
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * vmx_init() do the proper setup after enable_ept has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * established.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) vmentry_l1d_flush_param = l1tf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) mutex_lock(&vmx_l1d_flush_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ret = vmx_setup_l1d_flush(l1tf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) mutex_unlock(&vmx_l1d_flush_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return sprintf(s, "???\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) static const struct kernel_param_ops vmentry_l1d_flush_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) .set = vmentry_l1d_flush_set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) .get = vmentry_l1d_flush_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static u32 vmx_segment_access_rights(struct kvm_segment *var);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static __always_inline void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) u32 msr, int type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) void vmx_vmexit(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) #define vmx_insn_failed(fmt...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) WARN_ONCE(1, fmt); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) pr_warn_ratelimited(fmt); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) asmlinkage void vmread_error(unsigned long field, bool fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) kvm_spurious_fault();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) vmx_insn_failed("kvm: vmread failed: field=%lx\n", field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) noinline void vmwrite_error(unsigned long field, unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) ext, vpid, gva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) ext, eptp, gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static DEFINE_PER_CPU(struct vmcs *, vmxarea);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) DEFINE_PER_CPU(struct vmcs *, current_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) static DEFINE_SPINLOCK(vmx_vpid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) struct vmcs_config vmcs_config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct vmx_capability vmx_capability;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) #define VMX_SEGMENT_FIELD(seg) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) [VCPU_SREG_##seg] = { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) .selector = GUEST_##seg##_SELECTOR, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) .base = GUEST_##seg##_BASE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) .limit = GUEST_##seg##_LIMIT, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) .ar_bytes = GUEST_##seg##_AR_BYTES, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static const struct kvm_vmx_segment_field {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) unsigned selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) unsigned base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) unsigned limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) unsigned ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) } kvm_vmx_segment_fields[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) VMX_SEGMENT_FIELD(CS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) VMX_SEGMENT_FIELD(DS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) VMX_SEGMENT_FIELD(ES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) VMX_SEGMENT_FIELD(FS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) VMX_SEGMENT_FIELD(GS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) VMX_SEGMENT_FIELD(SS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) VMX_SEGMENT_FIELD(TR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) VMX_SEGMENT_FIELD(LDTR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) vmx->segment_cache.bitmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static unsigned long host_idt_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * will emulate SYSCALL in legacy mode if the vendor string in guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * support this emulation, IA32_STAR must always be included in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * vmx_uret_msrs_list[], even in i386 builds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static const u32 vmx_uret_msrs_list[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) MSR_EFER, MSR_TSC_AUX, MSR_STAR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) MSR_IA32_TSX_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) #if IS_ENABLED(CONFIG_HYPERV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) static bool __read_mostly enlightened_vmcs = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) module_param(enlightened_vmcs, bool, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /* check_ept_pointer() should be under protection of ept_pointer_lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static void check_ept_pointer_match(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) u64 tmp_eptp = INVALID_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) kvm_for_each_vcpu(i, vcpu, kvm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (!VALID_PAGE(tmp_eptp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) tmp_eptp = to_vmx(vcpu)->ept_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) to_kvm_vmx(kvm)->ept_pointers_match
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) = EPT_POINTERS_MISMATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct kvm_tlb_range *range = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) range->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct kvm_vcpu *vcpu, struct kvm_tlb_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) u64 ept_pointer = to_vmx(vcpu)->ept_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * of the base of EPT PML4 table, strip off EPT configuration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return hyperv_flush_guest_mapping_range(ept_pointer & PAGE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) kvm_fill_hv_flush_list_func, (void *)range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return hyperv_flush_guest_mapping(ept_pointer & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) static int hv_remote_flush_tlb_with_range(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct kvm_tlb_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) int ret = 0, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) check_ept_pointer_match(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) kvm_for_each_vcpu(i, vcpu, kvm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* If ept_pointer is invalid pointer, bypass flush request. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (VALID_PAGE(to_vmx(vcpu)->ept_pointer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ret |= __hv_remote_flush_tlb_with_range(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) kvm, vcpu, range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) ret = __hv_remote_flush_tlb_with_range(kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) kvm_get_vcpu(kvm, 0), range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static int hv_remote_flush_tlb(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return hv_remote_flush_tlb_with_range(kvm, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct hv_enlightened_vmcs *evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) struct hv_partition_assist_pg **p_hv_pa_pg =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) &vcpu->kvm->arch.hyperv.hv_pa_pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * Synthetic VM-Exit is not enabled in current code and so All
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * evmcs in singe VM shares same assist page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (!*p_hv_pa_pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!*p_hv_pa_pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) evmcs->partition_assist_page =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) __pa(*p_hv_pa_pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) evmcs->hv_vm_id = (unsigned long)vcpu->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) evmcs->hv_enlightenments_control.nested_flush_hypercall = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) #endif /* IS_ENABLED(CONFIG_HYPERV) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * Comment's format: document - errata name - stepping - processor name.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * Refer from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) static u32 vmx_preemption_cpu_tfms[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /* 323344.pdf - BA86 - D0 - Xeon 7500 Series */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) 0x000206E6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /* 323056.pdf - AAX65 - C2 - Xeon L3406 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* 322814.pdf - AAT59 - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) /* 322911.pdf - AAU65 - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) 0x00020652,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /* 322911.pdf - AAU65 - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) 0x00020655,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /* 322373.pdf - AAO95 - B1 - Xeon 3400 Series */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /* 322166.pdf - AAN92 - B1 - i7-800 and i5-700 Desktop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * 320767.pdf - AAP86 - B1 -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) 0x000106E5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) 0x000106A0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) 0x000106A1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) 0x000106A4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) 0x000106A5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /* Xeon E3-1220 V2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) 0x000306A8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) static inline bool cpu_has_broken_vmx_preemption_timer(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) u32 eax = cpuid_eax(0x00000001), i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /* Clear the reserved bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) eax &= ~(0x3U << 14 | 0xfU << 28);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (eax == vmx_preemption_cpu_tfms[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return flexpriority_enabled && lapic_in_kernel(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) static inline bool report_flexpriority(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return flexpriority_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) static int possible_passthrough_msr_slot(u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (vmx_possible_passthrough_msrs[i] == msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static bool is_valid_passthrough_msr(u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) bool r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) switch (msr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) case 0x800 ... 0x8ff:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /* x2APIC MSRs. These are handled in vmx_update_msr_bitmap_x2apic() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) case MSR_IA32_RTIT_STATUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) case MSR_IA32_RTIT_OUTPUT_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) case MSR_IA32_RTIT_OUTPUT_MASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) case MSR_IA32_RTIT_CR3_MATCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /* PT MSRs. These are handled in pt_update_intercept_for_msr() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) r = possible_passthrough_msr_slot(msr) != -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) WARN(!r, "Invalid MSR %x, please adapt vmx_possible_passthrough_msrs[]", msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) static inline int __vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) for (i = 0; i < vmx->nr_uret_msrs; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (vmx_uret_msrs_list[vmx->guest_uret_msrs[i].slot] == msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) i = __vmx_find_uret_msr(vmx, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (i >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return &vmx->guest_uret_msrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct vmx_uret_msr *msr, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) u64 old_msr_data = msr->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) msr->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (msr - vmx->guest_uret_msrs < vmx->nr_active_uret_msrs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) ret = kvm_set_user_return_msr(msr->slot, msr->data, msr->mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) msr->data = old_msr_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) #ifdef CONFIG_KEXEC_CORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static void crash_vmclear_local_loaded_vmcss(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct loaded_vmcs *v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) loaded_vmcss_on_cpu_link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) vmcs_clear(v->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) #endif /* CONFIG_KEXEC_CORE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) static void __loaded_vmcs_clear(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) struct loaded_vmcs *loaded_vmcs = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (loaded_vmcs->cpu != cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return; /* vcpu migration can race with cpu offline */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) per_cpu(current_vmcs, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) vmcs_clear(loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) vmcs_clear(loaded_vmcs->shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * Ensure all writes to loaded_vmcs, including deleting it from its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * current percpu list, complete before setting loaded_vmcs->vcpu to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * -1, otherwise a different cpu can see vcpu == -1 first and add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * loaded_vmcs to its percpu list before it's deleted from this cpu's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) loaded_vmcs->cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) loaded_vmcs->launched = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) int cpu = loaded_vmcs->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (cpu != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) smp_call_function_single(cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) __loaded_vmcs_clear, loaded_vmcs, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) unsigned field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) u32 mask = 1 << (seg * SEG_FIELD_NR + field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) vmx->segment_cache.bitmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) ret = vmx->segment_cache.bitmask & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) vmx->segment_cache.bitmask |= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) u16 *p = &vmx->segment_cache.seg[seg].selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) return *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) ulong *p = &vmx->segment_cache.seg[seg].base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) return *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) u32 *p = &vmx->segment_cache.seg[seg].limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) return *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) u32 *p = &vmx->segment_cache.seg[seg].ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) void update_exception_bitmap(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) u32 eb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) (1u << DB_VECTOR) | (1u << AC_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) * Guest access to VMware backdoor ports could legitimately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) * trigger #GP because of TSS I/O permission bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * We intercept those #GP and allow access to them anyway
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * as VMware does.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (enable_vmware_backdoor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) eb |= (1u << GP_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if ((vcpu->guest_debug &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) eb |= 1u << BP_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (to_vmx(vcpu)->rmode.vm86_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) eb = ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (!vmx_need_pf_intercept(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) eb &= ~(1u << PF_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) /* When we are running a nested L2 guest and L1 specified for it a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * certain exception bitmap, we must trap the same exceptions and pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * them to L1. When running L2, we will only handle the exceptions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * specified above if L1 did not want them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) eb |= get_vmcs12(vcpu)->exception_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * between guest and host. In that case we only care about present
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) * faults. For vmcs02, however, PFEC_MASK and PFEC_MATCH are set in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * prepare_vmcs02_rare.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) bool selective_pf_trap = enable_ept && (eb & (1u << PF_VECTOR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) int mask = selective_pf_trap ? PFERR_PRESENT_MASK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) vmcs_write32(EXCEPTION_BITMAP, eb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * Check if MSR is intercepted for currently loaded MSR bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) unsigned long *msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) msr_bitmap = vmx->loaded_vmcs->msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) if (msr <= 0x1fff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) return !!test_bit(msr, msr_bitmap + 0x800 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) msr &= 0x1fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) return !!test_bit(msr, msr_bitmap + 0xc00 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) unsigned long entry, unsigned long exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) vm_entry_controls_clearbit(vmx, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) vm_exit_controls_clearbit(vmx, exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) for (i = 0; i < m->nr; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if (m->val[i].index == msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) struct msr_autoload *m = &vmx->msr_autoload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) switch (msr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) case MSR_EFER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) if (cpu_has_load_ia32_efer()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) clear_atomic_switch_msr_special(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) VM_ENTRY_LOAD_IA32_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) VM_EXIT_LOAD_IA32_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) case MSR_CORE_PERF_GLOBAL_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) if (cpu_has_load_perf_global_ctrl()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) clear_atomic_switch_msr_special(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) i = vmx_find_loadstore_msr_slot(&m->guest, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (i < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) goto skip_guest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) --m->guest.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) m->guest.val[i] = m->guest.val[m->guest.nr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) skip_guest:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) i = vmx_find_loadstore_msr_slot(&m->host, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (i < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) --m->host.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) m->host.val[i] = m->host.val[m->host.nr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) unsigned long entry, unsigned long exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) u64 guest_val, u64 host_val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) vmcs_write64(guest_val_vmcs, guest_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (host_val_vmcs != HOST_IA32_EFER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) vmcs_write64(host_val_vmcs, host_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) vm_entry_controls_setbit(vmx, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) vm_exit_controls_setbit(vmx, exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) u64 guest_val, u64 host_val, bool entry_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) int i, j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) struct msr_autoload *m = &vmx->msr_autoload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) switch (msr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) case MSR_EFER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) if (cpu_has_load_ia32_efer()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) add_atomic_switch_msr_special(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) VM_ENTRY_LOAD_IA32_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) VM_EXIT_LOAD_IA32_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) GUEST_IA32_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) HOST_IA32_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) guest_val, host_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) case MSR_CORE_PERF_GLOBAL_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) if (cpu_has_load_perf_global_ctrl()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) add_atomic_switch_msr_special(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) GUEST_IA32_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) HOST_IA32_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) guest_val, host_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) case MSR_IA32_PEBS_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) /* PEBS needs a quiescent period after being disabled (to write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * a record). Disabling PEBS through VMX MSR swapping doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * provide that period, so a CPU could write host's record into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * guest's memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) i = vmx_find_loadstore_msr_slot(&m->guest, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (!entry_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) j = vmx_find_loadstore_msr_slot(&m->host, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if ((i < 0 && m->guest.nr == MAX_NR_LOADSTORE_MSRS) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) (j < 0 && m->host.nr == MAX_NR_LOADSTORE_MSRS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) printk_once(KERN_WARNING "Not enough msr switch entries. "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) "Can't add msr %x\n", msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (i < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) i = m->guest.nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) m->guest.val[i].index = msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) m->guest.val[i].value = guest_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (entry_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) if (j < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) j = m->host.nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) m->host.val[j].index = msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) m->host.val[j].value = host_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) static bool update_transition_efer(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) u64 guest_efer = vmx->vcpu.arch.efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) u64 ignore_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /* Shadow paging assumes NX to be available. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) guest_efer |= EFER_NX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) * LMA and LME handled by hardware; SCE meaningless outside long mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) ignore_bits |= EFER_SCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) ignore_bits |= EFER_LMA | EFER_LME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) /* SCE is meaningful only in long mode on Intel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (guest_efer & EFER_LMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) ignore_bits &= ~(u64)EFER_SCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * On EPT, we can't emulate NX, so we must switch EFER atomically.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * On CPUs that support "load IA32_EFER", always switch EFER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * atomically, since it's faster than switching it manually.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) if (cpu_has_load_ia32_efer() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (!(guest_efer & EFER_LMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) guest_efer &= ~EFER_LME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (guest_efer != host_efer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) add_atomic_switch_msr(vmx, MSR_EFER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) guest_efer, host_efer, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) clear_atomic_switch_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) i = __vmx_find_uret_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (i < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) clear_atomic_switch_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) guest_efer &= ~ignore_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) guest_efer |= host_efer & ignore_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) vmx->guest_uret_msrs[i].data = guest_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) vmx->guest_uret_msrs[i].mask = ~ignore_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * On 32-bit kernels, VM exits still load the FS and GS bases from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * VMCS rather than the segment table. KVM uses this helper to figure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) * out the current bases to poke them into the VMCS before entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) static unsigned long segment_base(u16 selector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) struct desc_struct *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) unsigned long v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (!(selector & ~SEGMENT_RPL_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) table = get_current_gdt_ro();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) u16 ldt_selector = kvm_read_ldt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) if (!(ldt_selector & ~SEGMENT_RPL_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) table = (struct desc_struct *)segment_base(ldt_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) v = get_desc_base(&table[selector >> 3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) return v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) static inline bool pt_can_write_msr(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) return vmx_pt_mode_is_host_guest() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) static inline bool pt_output_base_valid(struct kvm_vcpu *vcpu, u64 base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) /* The base must be 128-byte aligned and a legal physical address. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return !kvm_vcpu_is_illegal_gpa(vcpu, base) && !(base & 0x7f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) for (i = 0; i < addr_range; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) for (i = 0; i < addr_range; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) static void pt_guest_enter(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) if (vmx_pt_mode_is_system())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * GUEST_IA32_RTIT_CTL is already set in the VMCS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * Save host state before VM entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) wrmsrl(MSR_IA32_RTIT_CTL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) static void pt_guest_exit(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (vmx_pt_mode_is_system())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) unsigned long fs_base, unsigned long gs_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) if (unlikely(fs_sel != host->fs_sel)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) if (!(fs_sel & 7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) vmcs_write16(HOST_FS_SELECTOR, fs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) vmcs_write16(HOST_FS_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) host->fs_sel = fs_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) if (unlikely(gs_sel != host->gs_sel)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (!(gs_sel & 7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) vmcs_write16(HOST_GS_SELECTOR, gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) vmcs_write16(HOST_GS_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) host->gs_sel = gs_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (unlikely(fs_base != host->fs_base)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) vmcs_writel(HOST_FS_BASE, fs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) host->fs_base = fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (unlikely(gs_base != host->gs_base)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) vmcs_writel(HOST_GS_BASE, gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) host->gs_base = gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) struct vmcs_host_state *host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) unsigned long fs_base, gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) u16 fs_sel, gs_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) vmx->req_immediate_exit = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * Note that guest MSRs to be saved/restored can also be changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) * when guest state is loaded. This happens when guest transitions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) * to/from long-mode by setting MSR_EFER.LMA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) if (!vmx->guest_uret_msrs_loaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) vmx->guest_uret_msrs_loaded = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) for (i = 0; i < vmx->nr_active_uret_msrs; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) kvm_set_user_return_msr(vmx->guest_uret_msrs[i].slot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) vmx->guest_uret_msrs[i].data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) vmx->guest_uret_msrs[i].mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) if (vmx->nested.need_vmcs12_to_shadow_sync)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) nested_sync_vmcs12_to_shadow(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (vmx->guest_state_loaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) host_state = &vmx->loaded_vmcs->host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * allow segment selectors with cpl > 0 or ti == 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) host_state->ldt_sel = kvm_read_ldt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) savesegment(ds, host_state->ds_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) savesegment(es, host_state->es_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) gs_base = cpu_kernelmode_gs_base(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (likely(is_64bit_mm(current->mm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) current_save_fsgs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) fs_sel = current->thread.fsindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) gs_sel = current->thread.gsindex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) fs_base = current->thread.fsbase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) vmx->msr_host_kernel_gs_base = current->thread.gsbase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) savesegment(fs, fs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) savesegment(gs, gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) fs_base = read_msr(MSR_FS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) savesegment(fs, fs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) savesegment(gs, gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) fs_base = segment_base(fs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) gs_base = segment_base(gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) vmx->guest_state_loaded = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) struct vmcs_host_state *host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (!vmx->guest_state_loaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) host_state = &vmx->loaded_vmcs->host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) ++vmx->vcpu.stat.host_state_reload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) kvm_load_ldt(host_state->ldt_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) load_gs_index(host_state->gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) loadsegment(gs, host_state->gs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) if (host_state->fs_sel & 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) loadsegment(fs, host_state->fs_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) if (unlikely(host_state->ds_sel | host_state->es_sel)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) loadsegment(ds, host_state->ds_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) loadsegment(es, host_state->es_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) invalidate_tss_limit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) load_fixmap_gdt(raw_smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) vmx->guest_state_loaded = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) vmx->guest_uret_msrs_loaded = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) if (vmx->guest_state_loaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) return vmx->msr_guest_kernel_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (vmx->guest_state_loaded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) wrmsrl(MSR_KERNEL_GS_BASE, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) vmx->msr_guest_kernel_gs_base = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) struct loaded_vmcs *buddy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) struct vmcs *prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) if (!already_loaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) loaded_vmcs_clear(vmx->loaded_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * this cpu's percpu list, otherwise it may not yet be deleted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) * from its previous cpu's percpu list. Pairs with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * smb_wmb() in __loaded_vmcs_clear().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) &per_cpu(loaded_vmcss_on_cpu, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) prev = per_cpu(current_vmcs, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) if (prev != vmx->loaded_vmcs->vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) vmcs_load(vmx->loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) * No indirect branch prediction barrier needed when switching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) * the active VMCS within a guest, e.g. on nested VM-Enter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) * The L1 VMM can protect itself with retpolines, IBPB or IBRS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) indirect_branch_prediction_barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (!already_loaded) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) void *gdt = get_current_gdt_ro();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) unsigned long sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) * Flush all EPTP/VPID contexts, the new pCPU may have stale
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) * TLB entries from its previous association with the vCPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) * Linux uses per-cpu TSS and GDT, so set these when switching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) * processors. See 22.2.4.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) vmcs_writel(HOST_TR_BASE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) vmx->loaded_vmcs->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) /* Setup TSC multiplier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) if (kvm_has_tsc_control &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) decache_tsc_multiplier(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * Switches to specified vcpu, until a matching vcpu_put(), but assumes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) * vcpu mutex is already taken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) vmx_vcpu_load_vmcs(vcpu, cpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) vmx_vcpu_pi_load(vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) vmx->host_debugctlmsr = get_debugctlmsr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) vmx_vcpu_pi_put(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) vmx_prepare_switch_to_host(to_vmx(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) static bool emulation_required(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) return emulate_invalid_guest_state && !vmx_guest_state_valid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) unsigned long rflags, save_rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) rflags = vmcs_readl(GUEST_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (vmx->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) save_rflags = vmx->rmode.save_rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) vmx->rflags = rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return vmx->rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) unsigned long old_rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) if (is_unrestricted_guest(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) vmx->rflags = rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) vmcs_writel(GUEST_RFLAGS, rflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) old_rflags = vmx_get_rflags(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) vmx->rflags = rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) if (vmx->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) vmx->rmode.save_rflags = rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) vmcs_writel(GUEST_RFLAGS, rflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) vmx->emulation_required = emulation_required(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (interruptibility & GUEST_INTR_STATE_STI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) ret |= KVM_X86_SHADOW_INT_STI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) if (interruptibility & GUEST_INTR_STATE_MOV_SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) ret |= KVM_X86_SHADOW_INT_MOV_SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) u32 interruptibility = interruptibility_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) if (mask & KVM_X86_SHADOW_INT_MOV_SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) interruptibility |= GUEST_INTR_STATE_MOV_SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) else if (mask & KVM_X86_SHADOW_INT_STI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) interruptibility |= GUEST_INTR_STATE_STI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) if ((interruptibility != interruptibility_old))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) unsigned long value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) * Any MSR write that attempts to change bits marked reserved will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) * case a #GP fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) if (data & vmx->pt_desc.ctl_bitmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) * result in a #GP unless the same write also clears TraceEn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) * and FabricEn would cause #GP, if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) !(data & RTIT_CTL_FABRIC_EN) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) PT_CAP_single_range_output))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) * utilize encodings marked reserved will casue a #GP fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) !test_bit((data & RTIT_CTL_MTC_RANGE) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) RTIT_CTL_MTC_RANGE_OFFSET, &value))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) value = intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) PT_CAP_cycle_thresholds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) !test_bit((data & RTIT_CTL_CYC_THRESH) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) RTIT_CTL_CYC_THRESH_OFFSET, &value))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) !test_bit((data & RTIT_CTL_PSB_FREQ) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) RTIT_CTL_PSB_FREQ_OFFSET, &value))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) * If ADDRx_CFG is reserved or the encodings is >2 will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * cause a #GP fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) static bool vmx_can_emulate_instruction(struct kvm_vcpu *vcpu, void *insn, int insn_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) unsigned long rip, orig_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) * undefined behavior: Intel's SDM doesn't mandate the VMCS field be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * set when EPT misconfig occurs. In practice, real hardware updates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * (namely Hyper-V) don't set it due to it being undefined behavior,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * i.e. we end up advancing IP with some random value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) to_vmx(vcpu)->exit_reason.basic != EXIT_REASON_EPT_MISCONFIG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) orig_rip = kvm_rip_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) * We need to mask out the high 32 bits of RIP if not in 64-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) * mode, but just finding out that we are in 64-bit mode is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) * quite expensive. Only do it if there was a carry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) rip = (u32)rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) kvm_rip_write(vcpu, rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) /* skipping an emulated instruction also counts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) vmx_set_interrupt_shadow(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * Recognizes a pending MTF VM-exit and records the nested state for later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) * delivery.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) if (!is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * Per the SDM, MTF takes priority over debug-trap exceptions besides
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) * T-bit traps. As instruction emulation is completed (i.e. at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) * instruction boundary), any #DB exception pending delivery must be a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) * debug-trap. Record the pending MTF state to be delivered in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) * vmx_check_nested_events().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) if (nested_cpu_has_mtf(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) (!vcpu->arch.exception.pending ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) vcpu->arch.exception.nr == DB_VECTOR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) vmx->nested.mtf_pending = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) vmx->nested.mtf_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) vmx_update_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) return skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) * Ensure that we clear the HLT state in the VMCS. We don't need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) * explicitly skip the instruction because if the HLT state is set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) * then the instruction is already executing and RIP has already been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) * advanced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (kvm_hlt_in_guest(vcpu->kvm) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) static void vmx_queue_exception(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) unsigned nr = vcpu->arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) bool has_error_code = vcpu->arch.exception.has_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) u32 error_code = vcpu->arch.exception.error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) u32 intr_info = nr | INTR_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) kvm_deliver_exception_payload(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) if (has_error_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) intr_info |= INTR_INFO_DELIVER_CODE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) if (vmx->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) int inc_eip = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) if (kvm_exception_is_soft(nr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) inc_eip = vcpu->arch.event_exit_inst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) kvm_inject_realmode_interrupt(vcpu, nr, inc_eip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) WARN_ON_ONCE(vmx->emulation_required);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) if (kvm_exception_is_soft(nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) vmx->vcpu.arch.event_exit_inst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) intr_info |= INTR_TYPE_SOFT_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) intr_info |= INTR_TYPE_HARD_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) vmx_clear_hlt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) static void vmx_setup_uret_msr(struct vcpu_vmx *vmx, unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) struct vmx_uret_msr tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) int from, to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) from = __vmx_find_uret_msr(vmx, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) if (from < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) to = vmx->nr_active_uret_msrs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) tmp = vmx->guest_uret_msrs[to];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) vmx->guest_uret_msrs[to] = vmx->guest_uret_msrs[from];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) vmx->guest_uret_msrs[from] = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) * Set up the vmcs to automatically save and restore system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) * msrs. Don't touch the 64-bit msrs if the guest is in legacy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) * mode, as fiddling with msrs is very expensive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) static void setup_msrs(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) vmx->guest_uret_msrs_loaded = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) vmx->nr_active_uret_msrs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) * The SYSCALL MSRs are only needed on long mode guests, and only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * when EFER.SCE is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) if (is_long_mode(&vmx->vcpu) && (vmx->vcpu.arch.efer & EFER_SCE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) vmx_setup_uret_msr(vmx, MSR_STAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) vmx_setup_uret_msr(vmx, MSR_LSTAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) vmx_setup_uret_msr(vmx, MSR_SYSCALL_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) if (update_transition_efer(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) vmx_setup_uret_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) vmx_setup_uret_msr(vmx, MSR_TSC_AUX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) vmx_setup_uret_msr(vmx, MSR_IA32_TSX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) vmx_update_msr_bitmap(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) u64 g_tsc_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) * We're here if L1 chose not to trap WRMSR to TSC. According
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) * to the spec, this should set L1's TSC; The offset that L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) * set for L2 remains unchanged, and still needs to be added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * to the newly set TSC to get L2's TSC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) if (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) g_tsc_offset = vmcs12->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) trace_kvm_write_tsc_offset(vcpu->vcpu_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) vcpu->arch.tsc_offset - g_tsc_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) vmcs_write64(TSC_OFFSET, offset + g_tsc_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) return offset + g_tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) * all guests if the "nested" module option is off, and can also be disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) * for a single guest by disabling its VMX cpuid bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) uint64_t val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) return !(val & ~valid_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) switch (msr->index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) if (!nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) case MSR_IA32_PERF_CAPABILITIES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) msr->data = vmx_get_perf_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) return KVM_MSR_RET_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * Reads an msr value (of 'msr_index') into 'pdata'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * Returns 0 on success, non-0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * Assumes vcpu_load() was already called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) struct vmx_uret_msr *msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) switch (msr_info->index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) case MSR_FS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) msr_info->data = vmcs_readl(GUEST_FS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) case MSR_GS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) msr_info->data = vmcs_readl(GUEST_GS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) case MSR_KERNEL_GS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) case MSR_EFER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) return kvm_get_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) case MSR_IA32_TSX_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) goto find_uret_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) case MSR_IA32_UMWAIT_CONTROL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) msr_info->data = vmx->msr_ia32_umwait_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) case MSR_IA32_SPEC_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) !guest_has_spec_ctrl_msr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) msr_info->data = to_vmx(vcpu)->spec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) case MSR_IA32_SYSENTER_CS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) case MSR_IA32_SYSENTER_EIP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) case MSR_IA32_SYSENTER_ESP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) case MSR_IA32_BNDCFGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) if (!kvm_mpx_supported() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) msr_info->data = vmcs_read64(GUEST_BNDCFGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) case MSR_IA32_MCG_EXT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) !(vmx->msr_ia32_feature_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) FEAT_CTL_LMCE_ENABLED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) msr_info->data = vcpu->arch.mcg_ext_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) case MSR_IA32_FEAT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) msr_info->data = vmx->msr_ia32_feature_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) if (!nested_vmx_allowed(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) &msr_info->data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) * Enlightened VMCS v1 doesn't have certain VMCS fields but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) * instead of just ignoring the features, different Hyper-V
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) * versions are either trying to use them and fail or do some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) * sanity checking and refuse to boot. Filter all unsupported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * features out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) vmx->nested.enlightened_vmcs_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) nested_evmcs_filter_control_msr(msr_info->index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) &msr_info->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) case MSR_IA32_RTIT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) if (!vmx_pt_mode_is_host_guest())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) msr_info->data = vmx->pt_desc.guest.ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) case MSR_IA32_RTIT_STATUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) if (!vmx_pt_mode_is_host_guest())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) msr_info->data = vmx->pt_desc.guest.status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) case MSR_IA32_RTIT_CR3_MATCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) if (!vmx_pt_mode_is_host_guest() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) PT_CAP_cr3_filtering))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) msr_info->data = vmx->pt_desc.guest.cr3_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) case MSR_IA32_RTIT_OUTPUT_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) if (!vmx_pt_mode_is_host_guest() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) (!intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) PT_CAP_topa_output) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) PT_CAP_single_range_output)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) msr_info->data = vmx->pt_desc.guest.output_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) case MSR_IA32_RTIT_OUTPUT_MASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) if (!vmx_pt_mode_is_host_guest() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) (!intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) PT_CAP_topa_output) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) PT_CAP_single_range_output)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) msr_info->data = vmx->pt_desc.guest.output_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) if (!vmx_pt_mode_is_host_guest() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) PT_CAP_num_address_ranges)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) if (index % 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) msr_info->data = vmx->pt_desc.guest.addr_b[index / 2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) case MSR_TSC_AUX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) goto find_uret_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) find_uret_msr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) msr = vmx_find_uret_msr(vmx, msr_info->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) if (msr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) msr_info->data = msr->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) return kvm_get_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) return (u32)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) return (unsigned long)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * Writes msr value into the appropriate "register".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) * Returns 0 on success, non-0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) * Assumes vcpu_load() was already called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) struct vmx_uret_msr *msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) u32 msr_index = msr_info->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) u64 data = msr_info->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) switch (msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) case MSR_EFER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) ret = kvm_set_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) case MSR_FS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) vmx_segment_cache_clear(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) vmcs_writel(GUEST_FS_BASE, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) case MSR_GS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) vmx_segment_cache_clear(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) vmcs_writel(GUEST_GS_BASE, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) case MSR_KERNEL_GS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) vmx_write_guest_kernel_gs_base(vmx, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) case MSR_IA32_SYSENTER_CS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) get_vmcs12(vcpu)->guest_sysenter_cs = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) vmcs_write32(GUEST_SYSENTER_CS, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) case MSR_IA32_SYSENTER_EIP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) data = nested_vmx_truncate_sysenter_addr(vcpu, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) get_vmcs12(vcpu)->guest_sysenter_eip = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) vmcs_writel(GUEST_SYSENTER_EIP, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) case MSR_IA32_SYSENTER_ESP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) data = nested_vmx_truncate_sysenter_addr(vcpu, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) get_vmcs12(vcpu)->guest_sysenter_esp = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) vmcs_writel(GUEST_SYSENTER_ESP, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) case MSR_IA32_DEBUGCTLMSR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) VM_EXIT_SAVE_DEBUG_CONTROLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) get_vmcs12(vcpu)->guest_ia32_debugctl = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) ret = kvm_set_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) case MSR_IA32_BNDCFGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) if (!kvm_mpx_supported() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) (data & MSR_IA32_BNDCFGS_RSVD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) vmcs_write64(GUEST_BNDCFGS, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) case MSR_IA32_UMWAIT_CONTROL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) /* The reserved bit 1 and non-32 bit [63:32] should be zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) vmx->msr_ia32_umwait_control = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) case MSR_IA32_SPEC_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) !guest_has_spec_ctrl_msr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (kvm_spec_ctrl_test_value(data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) vmx->spec_ctrl = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) * For non-nested:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) * When it's written (to non-zero) for the first time, pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) * it through.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * For nested:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) * The handling of the MSR bitmap for L2 guests is done in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) * nested_vmx_prepare_msr_bitmap. We should not touch the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) * vmcs02.msr_bitmap here since it gets completely overwritten
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * in the merging. We update the vmcs01 here for L1 as well
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) * since it will end up touching the MSR anyway now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) vmx_disable_intercept_for_msr(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) MSR_IA32_SPEC_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) case MSR_IA32_TSX_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) goto find_uret_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) case MSR_IA32_PRED_CMD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) !guest_has_pred_cmd_msr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) if (data & ~PRED_CMD_IBPB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) if (!boot_cpu_has(X86_FEATURE_IBPB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) * For non-nested:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) * When it's written (to non-zero) for the first time, pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) * it through.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) * For nested:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) * The handling of the MSR bitmap for L2 guests is done in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) * nested_vmx_prepare_msr_bitmap. We should not touch the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) * vmcs02.msr_bitmap here since it gets completely overwritten
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) * in the merging.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) vmx_disable_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) case MSR_IA32_CR_PAT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) if (!kvm_pat_valid(data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) if (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) get_vmcs12(vcpu)->guest_ia32_pat = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) vmcs_write64(GUEST_IA32_PAT, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) vcpu->arch.pat = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) ret = kvm_set_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) case MSR_IA32_TSC_ADJUST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) ret = kvm_set_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) case MSR_IA32_MCG_EXT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) if ((!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) !(to_vmx(vcpu)->msr_ia32_feature_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) FEAT_CTL_LMCE_ENABLED)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) (data & ~MCG_EXT_CTL_LMCE_EN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) vcpu->arch.mcg_ext_ctl = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) case MSR_IA32_FEAT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) if (!vmx_feature_control_msr_valid(vcpu, data) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) (to_vmx(vcpu)->msr_ia32_feature_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) FEAT_CTL_LOCKED && !msr_info->host_initiated))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) vmx->msr_ia32_feature_control = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) if (msr_info->host_initiated && data == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) vmx_leave_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) if (!msr_info->host_initiated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) return 1; /* they are read-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) if (!nested_vmx_allowed(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) return vmx_set_vmx_msr(vcpu, msr_index, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) case MSR_IA32_RTIT_CTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) if (!vmx_pt_mode_is_host_guest() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) vmx_rtit_ctl_check(vcpu, data) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) vmx->nested.vmxon)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) vmcs_write64(GUEST_IA32_RTIT_CTL, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) vmx->pt_desc.guest.ctl = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) pt_update_intercept_for_msr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) case MSR_IA32_RTIT_STATUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) if (!pt_can_write_msr(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) if (data & MSR_IA32_RTIT_STATUS_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) vmx->pt_desc.guest.status = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) case MSR_IA32_RTIT_CR3_MATCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) if (!pt_can_write_msr(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) if (!intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) PT_CAP_cr3_filtering))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) vmx->pt_desc.guest.cr3_match = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) case MSR_IA32_RTIT_OUTPUT_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) if (!pt_can_write_msr(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) if (!intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) PT_CAP_topa_output) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) PT_CAP_single_range_output))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) if (!pt_output_base_valid(vcpu, data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) vmx->pt_desc.guest.output_base = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) case MSR_IA32_RTIT_OUTPUT_MASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) if (!pt_can_write_msr(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) if (!intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) PT_CAP_topa_output) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) !intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) PT_CAP_single_range_output))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) vmx->pt_desc.guest.output_mask = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) if (!pt_can_write_msr(vmx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) if (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) PT_CAP_num_address_ranges))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) if (is_noncanonical_address(data, vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) if (index % 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) vmx->pt_desc.guest.addr_b[index / 2] = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) vmx->pt_desc.guest.addr_a[index / 2] = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) case MSR_TSC_AUX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) if (!msr_info->host_initiated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) /* Check reserved bit, higher 32 bits should be zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) if ((data >> 32) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) goto find_uret_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) find_uret_msr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) msr = vmx_find_uret_msr(vmx, msr_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) if (msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) ret = vmx_set_guest_uret_msr(vmx, msr, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) ret = kvm_set_msr_common(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) unsigned long guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) kvm_register_mark_available(vcpu, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) switch (reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) case VCPU_REGS_RSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) case VCPU_REGS_RIP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) case VCPU_EXREG_PDPTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) ept_save_pdptrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) case VCPU_EXREG_CR0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) vcpu->arch.cr0 &= ~guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) case VCPU_EXREG_CR3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) if (is_unrestricted_guest(vcpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) (enable_ept && is_paging(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) case VCPU_EXREG_CR4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) vcpu->arch.cr4 &= ~guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) static __init int cpu_has_kvm_support(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) return cpu_has_vmx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) static __init int vmx_disabled_by_bios(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) !boot_cpu_has(X86_FEATURE_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) static int kvm_cpu_vmxon(u64 vmxon_pointer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) u64 msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) cr4_set_bits(X86_CR4_VMXE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) intel_pt_handle_vmx(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) _ASM_EXTABLE(1b, %l[fault])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) : : [vmxon_pointer] "m"(vmxon_pointer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) : : fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) intel_pt_handle_vmx(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) cr4_clear_bits(X86_CR4_VMXE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) static int hardware_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) if (cr4_read_shadow() & X86_CR4_VMXE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) * This can happen if we hot-added a CPU but failed to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) * VP assist page for it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) if (static_branch_unlikely(&enable_evmcs) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) !hv_get_vp_assist_page(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) r = kvm_cpu_vmxon(phys_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) ept_sync_global();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) static void vmclear_local_loaded_vmcss(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) struct loaded_vmcs *v, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) loaded_vmcss_on_cpu_link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) __loaded_vmcs_clear(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) * tricks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) static void kvm_cpu_vmxoff(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) asm volatile (__ex("vmxoff"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) intel_pt_handle_vmx(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) cr4_clear_bits(X86_CR4_VMXE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) static void hardware_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) vmclear_local_loaded_vmcss();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) kvm_cpu_vmxoff();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) * directly instead of going through cpu_has(), to ensure KVM is trapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) * ENCLS whenever it's supported in hardware. It does not matter whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) * the host OS supports or has enabled SGX.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) static bool cpu_has_sgx(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) u32 msr, u32 *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) u32 vmx_msr_low, vmx_msr_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) u32 ctl = ctl_min | ctl_opt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) rdmsr(msr, vmx_msr_low, vmx_msr_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) /* Ensure minimum (required) set of control bits are supported. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (ctl_min & ~ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) *result = ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) struct vmx_capability *vmx_cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) u32 vmx_msr_low, vmx_msr_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) u32 min, opt, min2, opt2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) u32 _pin_based_exec_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) u32 _cpu_based_exec_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) u32 _cpu_based_2nd_exec_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) u32 _vmexit_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) u32 _vmentry_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) memset(vmcs_conf, 0, sizeof(*vmcs_conf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) min = CPU_BASED_HLT_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) CPU_BASED_CR8_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) CPU_BASED_CR8_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) CPU_BASED_CR3_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) CPU_BASED_UNCOND_IO_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) CPU_BASED_MOV_DR_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) CPU_BASED_USE_TSC_OFFSETTING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) CPU_BASED_MWAIT_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) CPU_BASED_MONITOR_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) CPU_BASED_INVLPG_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) CPU_BASED_RDPMC_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) opt = CPU_BASED_TPR_SHADOW |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) CPU_BASED_USE_MSR_BITMAPS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) &_cpu_based_exec_control) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) ~CPU_BASED_CR8_STORE_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) min2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) SECONDARY_EXEC_WBINVD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) SECONDARY_EXEC_ENABLE_VPID |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) SECONDARY_EXEC_ENABLE_EPT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) SECONDARY_EXEC_UNRESTRICTED_GUEST |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) SECONDARY_EXEC_PAUSE_LOOP_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) SECONDARY_EXEC_DESC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) SECONDARY_EXEC_ENABLE_RDTSCP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) SECONDARY_EXEC_ENABLE_INVPCID |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) SECONDARY_EXEC_SHADOW_VMCS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) SECONDARY_EXEC_XSAVES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) SECONDARY_EXEC_RDSEED_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) SECONDARY_EXEC_RDRAND_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) SECONDARY_EXEC_ENABLE_PML |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) SECONDARY_EXEC_TSC_SCALING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) SECONDARY_EXEC_PT_USE_GPA |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) SECONDARY_EXEC_PT_CONCEAL_VMX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) SECONDARY_EXEC_ENABLE_VMFUNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) if (cpu_has_sgx())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) opt2 |= SECONDARY_EXEC_ENCLS_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) if (adjust_vmx_controls(min2, opt2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) MSR_IA32_VMX_PROCBASED_CTLS2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) &_cpu_based_2nd_exec_control) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) #ifndef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) if (!(_cpu_based_2nd_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) _cpu_based_2nd_exec_control &= ~(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) &vmx_cap->ept, &vmx_cap->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) CPU_BASED_CR3_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) CPU_BASED_INVLPG_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) } else if (vmx_cap->ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) vmx_cap->ept = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) pr_warn_once("EPT CAP should not exist if not support "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) "1-setting enable EPT VM-execution control\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) vmx_cap->vpid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) vmx_cap->vpid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) pr_warn_once("VPID CAP should not exist if not support "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) "1-setting enable VPID VM-execution control\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) VM_EXIT_LOAD_IA32_PAT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) VM_EXIT_LOAD_IA32_EFER |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) VM_EXIT_CLEAR_BNDCFGS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) VM_EXIT_PT_CONCEAL_PIP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) VM_EXIT_CLEAR_IA32_RTIT_CTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) &_vmexit_control) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) PIN_BASED_VMX_PREEMPTION_TIMER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) &_pin_based_exec_control) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) if (cpu_has_broken_vmx_preemption_timer())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) if (!(_cpu_based_2nd_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) VM_ENTRY_LOAD_IA32_PAT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) VM_ENTRY_LOAD_IA32_EFER |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) VM_ENTRY_LOAD_BNDCFGS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) VM_ENTRY_PT_CONCEAL_PIP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) VM_ENTRY_LOAD_IA32_RTIT_CTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) &_vmentry_control) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) * can't be used due to an errata where VM Exit may incorrectly clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) if (boot_cpu_data.x86 == 0x6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) switch (boot_cpu_data.x86_model) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) case 26: /* AAK155 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) case 30: /* AAP115 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) case 37: /* AAT100 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) case 44: /* BC86,AAY89,BD102 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) case 46: /* BA97 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) "does not work properly. Using workaround\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) if (vmx_msr_high & (1u<<16))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) /* Require Write-Back (WB) memory type for VMCS accesses. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) if (((vmx_msr_high >> 18) & 15) != 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) vmcs_conf->size = vmx_msr_high & 0x1fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) vmcs_conf->order = get_order(vmcs_conf->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) vmcs_conf->revision_id = vmx_msr_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) vmcs_conf->vmexit_ctrl = _vmexit_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) vmcs_conf->vmentry_ctrl = _vmentry_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) #if IS_ENABLED(CONFIG_HYPERV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) if (enlightened_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) evmcs_sanitize_exec_ctrls(vmcs_conf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) int node = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) struct page *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) struct vmcs *vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) pages = __alloc_pages_node(node, flags, vmcs_config.order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) if (!pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) vmcs = page_address(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) memset(vmcs, 0, vmcs_config.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) /* KVM supports Enlightened VMCS v1 only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) vmcs->hdr.revision_id = vmcs_config.revision_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) if (shadow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) vmcs->hdr.shadow_vmcs = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) return vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) void free_vmcs(struct vmcs *vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) free_pages((unsigned long)vmcs, vmcs_config.order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) if (!loaded_vmcs->vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) loaded_vmcs_clear(loaded_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) free_vmcs(loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) loaded_vmcs->vmcs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) if (loaded_vmcs->msr_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) free_page((unsigned long)loaded_vmcs->msr_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) loaded_vmcs->vmcs = alloc_vmcs(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) if (!loaded_vmcs->vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) vmcs_clear(loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) loaded_vmcs->shadow_vmcs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) loaded_vmcs->hv_timer_soft_disabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) loaded_vmcs->cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) loaded_vmcs->launched = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) if (cpu_has_vmx_msr_bitmap()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) loaded_vmcs->msr_bitmap = (unsigned long *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) __get_free_page(GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) if (!loaded_vmcs->msr_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) goto out_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) if (IS_ENABLED(CONFIG_HYPERV) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) static_branch_unlikely(&enable_evmcs) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) struct hv_enlightened_vmcs *evmcs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) evmcs->hv_enlightenments_control.msr_bitmap = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) memset(&loaded_vmcs->controls_shadow, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) sizeof(struct vmcs_controls_shadow));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) out_vmcs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) free_loaded_vmcs(loaded_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) static void free_kvm_area(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) free_vmcs(per_cpu(vmxarea, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) per_cpu(vmxarea, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) static __init int alloc_kvm_area(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) struct vmcs *vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) if (!vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) free_kvm_area();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) * When eVMCS is enabled, alloc_vmcs_cpu() sets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) * vmcs->revision_id to KVM_EVMCS_VERSION instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) * revision_id reported by MSR_IA32_VMX_BASIC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) * However, even though not explicitly documented by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) * TLFS, VMXArea passed as VMXON argument should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) * still be marked with revision_id reported by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) * physical CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) vmcs->hdr.revision_id = vmcs_config.revision_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) per_cpu(vmxarea, cpu) = vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) struct kvm_segment *save)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) if (!emulate_invalid_guest_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) * CS and SS RPL should be equal during guest entry according
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) * to VMX spec, but in reality it is not always so. Since vcpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) * is in the middle of the transition from real mode to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) * protected mode it is safe to assume that RPL 0 is a good
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) * default value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) save->selector &= ~SEGMENT_RPL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) save->dpl = save->selector & SEGMENT_RPL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) save->s = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) vmx_set_segment(vcpu, save, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) static void enter_pmode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) * Update real mode segment cache. It may be not up-to-date if sement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) * register was written while vcpu was in a guest mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) vmx->rmode.vm86_active = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) flags = vmcs_readl(GUEST_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) vmcs_writel(GUEST_RFLAGS, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) update_exception_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) static void fix_rmode_seg(int seg, struct kvm_segment *save)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) struct kvm_segment var = *save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) var.dpl = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) if (seg == VCPU_SREG_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) var.type = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) if (!emulate_invalid_guest_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) var.selector = var.base >> 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) var.base = var.base & 0xffff0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) var.limit = 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) var.g = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) var.db = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) var.present = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) var.s = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) var.l = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) var.unusable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) var.type = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) var.avl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) if (save->base & 0xf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) printk_once(KERN_WARNING "kvm: segment base is not "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) "paragraph aligned when entering "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) "protected mode (seg=%d)", seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) vmcs_write16(sf->selector, var.selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) vmcs_writel(sf->base, var.base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) vmcs_write32(sf->limit, var.limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) static void enter_rmode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) vmx->rmode.vm86_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) * Very old userspace does not call KVM_SET_TSS_ADDR before entering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) * vcpu. Warn the user that an update is overdue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) if (!kvm_vmx->tss_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) "called before entering vcpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) vmx_segment_cache_clear(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) flags = vmcs_readl(GUEST_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) vmx->rmode.save_rflags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) vmcs_writel(GUEST_RFLAGS, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) update_exception_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) kvm_mmu_reset_context(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) struct vmx_uret_msr *msr = vmx_find_uret_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) /* Nothing to do if hardware doesn't support EFER. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) if (!msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) vcpu->arch.efer = efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) if (efer & EFER_LMA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) msr->data = efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) msr->data = efer & ~EFER_LME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) setup_msrs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) static void enter_lmode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) u32 guest_tr_ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) vmx_segment_cache_clear(to_vmx(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) pr_debug_ratelimited("%s: tss fixup for long mode. \n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) vmcs_write32(GUEST_TR_AR_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) (guest_tr_ar & ~VMX_AR_TYPE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) | VMX_AR_TYPE_BUSY_64_TSS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) static void exit_lmode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * INVEPT must be issued when EPT is enabled, irrespective of VPID, as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * the CPU is not required to invalidate guest-physical mappings on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) * VM-Entry, even if VPID is disabled. Guest-physical mappings are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) * associated with the root EPT structure and not any particular VPID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) * (INVVPID also isn't required to invalidate guest-physical mappings).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) ept_sync_global();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) } else if (enable_vpid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) if (cpu_has_vmx_invvpid_global()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) vpid_sync_vcpu_global();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) vpid_sync_vcpu_single(vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) vpid_sync_vcpu_single(vmx->nested.vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) return nested_get_vpid02(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) return to_vmx(vcpu)->vpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) struct kvm_mmu *mmu = vcpu->arch.mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) u64 root_hpa = mmu->root_hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) /* No flush required if the current context is invalid. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) if (!VALID_PAGE(root_hpa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) ept_sync_context(construct_eptp(vcpu, root_hpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) mmu->shadow_root_level));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) vpid_sync_context(vmx_get_current_vpid(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) * vmx_flush_tlb_guest() for an explanation of why this is ok.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) * vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) * required to flush GVA->{G,H}PA mappings from the TLB if vpid is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) * i.e. no explicit INVVPID is necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) vpid_sync_context(vmx_get_current_vpid(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) if (is_pae_paging(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) void ept_save_pdptrs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) if (WARN_ON_ONCE(!is_pae_paging(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) unsigned long cr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) vmx_cache_reg(vcpu, VCPU_EXREG_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) if (!(cr0 & X86_CR0_PG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) /* From paging/starting to nonpaging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) exec_controls_setbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) CPU_BASED_CR3_STORE_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) vcpu->arch.cr0 = cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) } else if (!is_paging(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) /* From nonpaging to paging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) exec_controls_clearbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) CPU_BASED_CR3_STORE_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) vcpu->arch.cr0 = cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) if (!(cr0 & X86_CR0_WP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) *hw_cr0 &= ~X86_CR0_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) unsigned long hw_cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) if (is_unrestricted_guest(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) enter_pmode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) enter_rmode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) if (vcpu->arch.efer & EFER_LME) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) enter_lmode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) exit_lmode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) if (enable_ept && !is_unrestricted_guest(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) vmcs_writel(CR0_READ_SHADOW, cr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) vmcs_writel(GUEST_CR0, hw_cr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) vcpu->arch.cr0 = cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) kvm_register_mark_available(vcpu, VCPU_EXREG_CR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) /* depends on vcpu->arch.cr0 to be set to a new value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) vmx->emulation_required = emulation_required(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) static int vmx_get_max_tdp_level(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (cpu_has_vmx_ept_5levels())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) return 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) return 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) int root_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) u64 eptp = VMX_EPTP_MT_WB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) if (enable_ept_ad_bits &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) eptp |= VMX_EPTP_AD_ENABLE_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) eptp |= (root_hpa & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) return eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) int pgd_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) struct kvm *kvm = vcpu->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) bool update_guest_cr3 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) unsigned long guest_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) u64 eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) eptp = construct_eptp(vcpu, pgd, pgd_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) vmcs_write64(EPT_POINTER, eptp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) if (kvm_x86_ops.tlb_remote_flush) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) to_vmx(vcpu)->ept_pointer = eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) to_kvm_vmx(kvm)->ept_pointers_match
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) = EPT_POINTERS_CHECK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) if (!enable_unrestricted_guest && !is_paging(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) guest_cr3 = vcpu->arch.cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) else /* vmcs01.GUEST_CR3 is already up-to-date. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) update_guest_cr3 = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) vmx_ept_load_pdptrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) guest_cr3 = pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) if (update_guest_cr3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) vmcs_writel(GUEST_CR3, guest_cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) * Pass through host's Machine Check Enable value to hw_cr4, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) * is in force while we are in guest mode. Do not let guests control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) * this bit, even if host CR4.MCE == 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) unsigned long hw_cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) if (is_unrestricted_guest(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) else if (vmx->rmode.vm86_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) if (cr4 & X86_CR4_UMIP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) hw_cr4 &= ~X86_CR4_UMIP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) } else if (!is_guest_mode(vcpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) if (cr4 & X86_CR4_VMXE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) * To use VMXON (and later other VMX instructions), a guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) * must first be able to turn on cr4.VMXE (see handle_vmon()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) * So basically the check on whether to allow nested VMX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) * is here. We operate under the default treatment of SMM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) * so VMX cannot be enabled under SMM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) if (vmx->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) vcpu->arch.cr4 = cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) kvm_register_mark_available(vcpu, VCPU_EXREG_CR4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) if (!is_unrestricted_guest(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) if (!is_paging(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) hw_cr4 &= ~X86_CR4_PAE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) hw_cr4 |= X86_CR4_PSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) } else if (!(cr4 & X86_CR4_PAE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) hw_cr4 &= ~X86_CR4_PAE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) * hardware. To emulate this behavior, SMEP/SMAP/PKU needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) * to be manually disabled when guest switches to non-paging
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) * mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) * If !enable_unrestricted_guest, the CPU is always running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) * with CR0.PG=1 and CR4 needs to be modified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) * If enable_unrestricted_guest, the CPU automatically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) if (!is_paging(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) vmcs_writel(CR4_READ_SHADOW, cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) vmcs_writel(GUEST_CR4, hw_cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) u32 ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) *var = vmx->rmode.segs[seg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) if (seg == VCPU_SREG_TR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) || var->selector == vmx_read_guest_seg_selector(vmx, seg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) var->base = vmx_read_guest_seg_base(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) var->selector = vmx_read_guest_seg_selector(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) var->base = vmx_read_guest_seg_base(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) var->limit = vmx_read_guest_seg_limit(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) var->selector = vmx_read_guest_seg_selector(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) ar = vmx_read_guest_seg_ar(vmx, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) var->unusable = (ar >> 16) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) var->type = ar & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) var->s = (ar >> 4) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) var->dpl = (ar >> 5) & 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) * Some userspaces do not preserve unusable property. Since usable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) * segment has to be present according to VMX spec we can use present
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) * property to amend userspace bug by making unusable segment always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * nonpresent. vmx_segment_access_rights() already marks nonpresent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * segment as unusable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) var->present = !var->unusable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) var->avl = (ar >> 12) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) var->l = (ar >> 13) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) var->db = (ar >> 14) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) var->g = (ar >> 15) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) struct kvm_segment s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) if (to_vmx(vcpu)->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) vmx_get_segment(vcpu, &s, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) return s.base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) int vmx_get_cpl(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) if (unlikely(vmx->rmode.vm86_active))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) return VMX_AR_DPL(ar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) static u32 vmx_segment_access_rights(struct kvm_segment *var)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) u32 ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) if (var->unusable || !var->present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) ar = 1 << 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) ar = var->type & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) ar |= (var->s & 1) << 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) ar |= (var->dpl & 3) << 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) ar |= (var->present & 1) << 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) ar |= (var->avl & 1) << 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) ar |= (var->l & 1) << 13;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) ar |= (var->db & 1) << 14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) ar |= (var->g & 1) << 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) return ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) vmx_segment_cache_clear(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) vmx->rmode.segs[seg] = *var;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) if (seg == VCPU_SREG_TR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) vmcs_write16(sf->selector, var->selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) else if (var->s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) vmcs_writel(sf->base, var->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) vmcs_write32(sf->limit, var->limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) vmcs_write16(sf->selector, var->selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) * Fix the "Accessed" bit in AR field of segment registers for older
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) * qemu binaries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) * IA32 arch specifies that at the time of processor reset the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) * "Accessed" bit in the AR field of segment registers is 1. And qemu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) * is setting it to 0 in the userland code. This causes invalid guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) * state vmexit when "unrestricted guest" mode is turned on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) * Fix for this setup issue in cpu_reset is being pushed in the qemu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) * tree. Newer qemu binaries with that qemu fix would not need this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) * kvm hack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) if (is_unrestricted_guest(vcpu) && (seg != VCPU_SREG_LDTR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) var->type |= 0x1; /* Accessed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) vmx->emulation_required = emulation_required(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) *db = (ar >> 14) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) *l = (ar >> 13) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) dt->address = vmcs_readl(GUEST_IDTR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) vmcs_writel(GUEST_IDTR_BASE, dt->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) dt->address = vmcs_readl(GUEST_GDTR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) vmcs_writel(GUEST_GDTR_BASE, dt->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) struct kvm_segment var;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) u32 ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) vmx_get_segment(vcpu, &var, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) var.dpl = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) if (seg == VCPU_SREG_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) var.type = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) ar = vmx_segment_access_rights(&var);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) if (var.base != (var.selector << 4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) if (var.limit != 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) if (ar != 0xf3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) static bool code_segment_valid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) struct kvm_segment cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) unsigned int cs_rpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) cs_rpl = cs.selector & SEGMENT_RPL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) if (cs.unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) if (!cs.s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) if (cs.dpl > cs_rpl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) if (cs.dpl != cs_rpl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) if (!cs.present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) static bool stack_segment_valid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) struct kvm_segment ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) unsigned int ss_rpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) ss_rpl = ss.selector & SEGMENT_RPL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) if (ss.unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) if (ss.type != 3 && ss.type != 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) if (!ss.s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) if (ss.dpl != ss_rpl) /* DPL != RPL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) if (!ss.present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) struct kvm_segment var;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) unsigned int rpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) vmx_get_segment(vcpu, &var, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) rpl = var.selector & SEGMENT_RPL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) if (var.unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) if (!var.s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) if (!var.present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) if (var.dpl < rpl) /* DPL < RPL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) /* TODO: Add other members to kvm_segment_field to allow checking for other access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) * rights flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) static bool tr_valid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) struct kvm_segment tr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) if (tr.unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) if (tr.selector & SEGMENT_TI_MASK) /* TI = 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) if (!tr.present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) static bool ldtr_valid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) struct kvm_segment ldtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) if (ldtr.unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) if (ldtr.selector & SEGMENT_TI_MASK) /* TI = 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) if (ldtr.type != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) if (!ldtr.present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) struct kvm_segment cs, ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) return ((cs.selector & SEGMENT_RPL_MASK) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) (ss.selector & SEGMENT_RPL_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) * Check if guest state is valid. Returns true if valid, false if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) * not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) * We assume that registers are always usable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) /* real mode guest state checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) /* protected mode guest state checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) if (!cs_ss_rpl_check(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) if (!code_segment_valid(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) if (!stack_segment_valid(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) if (!data_segment_valid(vcpu, VCPU_SREG_DS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) if (!data_segment_valid(vcpu, VCPU_SREG_ES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) if (!data_segment_valid(vcpu, VCPU_SREG_FS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) if (!data_segment_valid(vcpu, VCPU_SREG_GS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) if (!tr_valid(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) if (!ldtr_valid(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) /* TODO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) * - Add checks on RIP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) * - Add checks on RFLAGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) static int init_rmode_tss(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) gfn_t fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) u16 data = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) int idx, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) r = kvm_write_guest_page(kvm, fn++, &data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) TSS_IOPB_BASE_OFFSET, sizeof(u16));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) data = ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) r = kvm_write_guest_page(kvm, fn, &data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) sizeof(u8));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) static int init_rmode_identity_map(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) int i, r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) kvm_pfn_t identity_map_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) u32 tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) /* Protect kvm_vmx->ept_identity_pagetable_done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) if (likely(kvm_vmx->ept_identity_pagetable_done))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) if (!kvm_vmx->ept_identity_map_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) kvm_vmx->ept_identity_map_addr, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) /* Set up identity-mapping pagetable for EPT in real mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) r = kvm_write_guest_page(kvm, identity_map_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) &tmp, i * sizeof(tmp), sizeof(tmp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) kvm_vmx->ept_identity_pagetable_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) static void seg_setup(int seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) unsigned int ar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) vmcs_write16(sf->selector, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) vmcs_writel(sf->base, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) vmcs_write32(sf->limit, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) ar = 0x93;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) if (seg == VCPU_SREG_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) ar |= 0x08; /* code segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) vmcs_write32(sf->ar_bytes, ar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) static int alloc_apic_access_page(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) if (kvm->arch.apic_access_page_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) if (is_error_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) * Do not pin the page in memory, so that memory hot-unplug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) * is able to migrate it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) kvm->arch.apic_access_page_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) int allocate_vpid(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) int vpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) if (!enable_vpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) spin_lock(&vmx_vpid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) if (vpid < VMX_NR_VPIDS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) __set_bit(vpid, vmx_vpid_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) vpid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) spin_unlock(&vmx_vpid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) return vpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) void free_vpid(int vpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) if (!enable_vpid || vpid == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) spin_lock(&vmx_vpid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) __clear_bit(vpid, vmx_vpid_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) spin_unlock(&vmx_vpid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) static void vmx_clear_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) if (msr <= 0x1fff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) __clear_bit(msr, msr_bitmap + 0x000 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) __clear_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) static void vmx_clear_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) if (msr <= 0x1fff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) __clear_bit(msr, msr_bitmap + 0x800 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) __clear_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) static void vmx_set_msr_bitmap_read(ulong *msr_bitmap, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) if (msr <= 0x1fff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) __set_bit(msr, msr_bitmap + 0x000 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) __set_bit(msr & 0x1fff, msr_bitmap + 0x400 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) static void vmx_set_msr_bitmap_write(ulong *msr_bitmap, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) if (msr <= 0x1fff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) __set_bit(msr, msr_bitmap + 0x800 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) __set_bit(msr & 0x1fff, msr_bitmap + 0xc00 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) static __always_inline void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) u32 msr, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) if (!cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) evmcs_touch_msr_bitmap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) * Mark the desired intercept state in shadow bitmap, this is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) * for resync when the MSR filters change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) if (is_valid_passthrough_msr(msr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) int idx = possible_passthrough_msr_slot(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) if (idx != -ENOENT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) if (type & MSR_TYPE_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) clear_bit(idx, vmx->shadow_msr_intercept.read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) if (type & MSR_TYPE_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) clear_bit(idx, vmx->shadow_msr_intercept.write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) if ((type & MSR_TYPE_R) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) vmx_set_msr_bitmap_read(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) type &= ~MSR_TYPE_R;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) if ((type & MSR_TYPE_W) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) !kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) vmx_set_msr_bitmap_write(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) type &= ~MSR_TYPE_W;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) if (type & MSR_TYPE_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) vmx_clear_msr_bitmap_read(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) if (type & MSR_TYPE_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) vmx_clear_msr_bitmap_write(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) static __always_inline void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) u32 msr, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) if (!cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) evmcs_touch_msr_bitmap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) * Mark the desired intercept state in shadow bitmap, this is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) * for resync when the MSR filter changes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) if (is_valid_passthrough_msr(msr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) int idx = possible_passthrough_msr_slot(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) if (idx != -ENOENT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) if (type & MSR_TYPE_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) set_bit(idx, vmx->shadow_msr_intercept.read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) if (type & MSR_TYPE_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) set_bit(idx, vmx->shadow_msr_intercept.write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) if (type & MSR_TYPE_R)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) vmx_set_msr_bitmap_read(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) if (type & MSR_TYPE_W)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) vmx_set_msr_bitmap_write(msr_bitmap, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) static __always_inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) u32 msr, int type, bool value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) if (value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) vmx_enable_intercept_for_msr(vcpu, msr, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) vmx_disable_intercept_for_msr(vcpu, msr, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) u8 mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) if (cpu_has_secondary_exec_ctrls() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) (secondary_exec_controls_get(to_vmx(vcpu)) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) mode |= MSR_BITMAP_MODE_X2APIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) mode |= MSR_BITMAP_MODE_X2APIC_APICV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) static void vmx_reset_x2apic_msrs(struct kvm_vcpu *vcpu, u8 mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) unsigned long *msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) unsigned long read_intercept;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) int msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) read_intercept = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) unsigned int read_idx = msr / BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) unsigned int write_idx = read_idx + (0x800 / sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) msr_bitmap[read_idx] = read_intercept;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) msr_bitmap[write_idx] = ~0ul;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) static void vmx_update_msr_bitmap_x2apic(struct kvm_vcpu *vcpu, u8 mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) if (!cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) vmx_reset_x2apic_msrs(vcpu, mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) * TPR reads and writes can be virtualized even if virtual interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) * delivery is not in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) vmx_set_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) !(mode & MSR_BITMAP_MODE_X2APIC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) vmx_enable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) vmx_disable_intercept_for_msr(vcpu, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) u8 mode = vmx_msr_bitmap_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) u8 changed = mode ^ vmx->msr_bitmap_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) if (!changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) vmx_update_msr_bitmap_x2apic(vcpu, mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) vmx->msr_bitmap_mode = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) bool flag = !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_STATUS, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_BASE, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_OUTPUT_MASK, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_CR3_MATCH, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) for (i = 0; i < vmx->pt_desc.addr_range; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_A + i * 2, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) vmx_set_intercept_for_msr(vcpu, MSR_IA32_RTIT_ADDR0_B + i * 2, MSR_TYPE_RW, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) void *vapic_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) u32 vppr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) int rvi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) WARN_ON_ONCE(!vmx->nested.virtual_apic_map.gfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) rvi = vmx_get_rvi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) vapic_page = vmx->nested.virtual_apic_map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) return ((rvi & 0xf0) > (vppr & 0xf0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) * Set intercept permissions for all potentially passed through MSRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) * again. They will automatically get filtered through the MSR filter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) * so we are back in sync after this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) for (i = 0; i < ARRAY_SIZE(vmx_possible_passthrough_msrs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) u32 msr = vmx_possible_passthrough_msrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) bool read = test_bit(i, vmx->shadow_msr_intercept.read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) bool write = test_bit(i, vmx->shadow_msr_intercept.write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_R, read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) vmx_set_intercept_for_msr(vcpu, msr, MSR_TYPE_W, write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) pt_update_intercept_for_msr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) vmx_update_msr_bitmap_x2apic(vcpu, vmx_msr_bitmap_mode(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) bool nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) if (vcpu->mode == IN_GUEST_MODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) * The vector of interrupt to be delivered to vcpu had
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) * been set in PIR before this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) * Following cases will be reached in this block, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) * we always send a notification event in all cases as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) * explained below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) * Case 1: vcpu keeps in non-root mode. Sending a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) * notification event posts the interrupt to vcpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) * Case 2: vcpu exits to root mode and is still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) * runnable. PIR will be synced to vIRR before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) * next vcpu entry. Sending a notification event in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) * this case has no effect, as vcpu is not in root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) * mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) * Case 3: vcpu exits to root mode and is blocked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) * vcpu_block() has already synced PIR to vIRR and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) * never blocks vcpu if vIRR is not cleared. Therefore,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) * a blocked vcpu here does not wait for any requested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) * interrupts in PIR, and sending a notification event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) * which has no effect is safe here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) int vector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) if (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) vector == vmx->nested.posted_intr_nv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) * If a posted intr is not recognized by hardware,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) * we will accomplish it in the next vmentry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) vmx->nested.pi_pending = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) /* the PIR and ON have been set by L1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) kvm_vcpu_kick(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) * Send interrupt to vcpu via posted interrupt way.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) * 1. If target vcpu is running(non-root mode), send posted interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) * notification to vcpu and hardware will sync PIR to vIRR atomically.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) * 2. If target vcpu isn't running(root mode), kick it to pick up the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) * interrupt from PIR in next vmentry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) if (!r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) if (!vcpu->arch.apicv_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) if (pi_test_and_set_pir(vector, &vmx->pi_desc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) /* If a previous notification has sent the IPI, nothing to do. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) if (pi_test_and_set_on(&vmx->pi_desc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) kvm_vcpu_kick(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) * Set up the vmcs's constant host-state fields, i.e., host-state fields that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) * will not change in the lifetime of the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) * Note that host-state that does change is set elsewhere. E.g., host-state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) u32 low32, high32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) unsigned long tmpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) unsigned long cr0, cr3, cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) cr0 = read_cr0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) WARN_ON(cr0 & X86_CR0_TS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) vmcs_writel(HOST_CR0, cr0); /* 22.2.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) * Save the most likely value for this task's CR3 in the VMCS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) * We can't use __get_current_cr3_fast() because we're not atomic.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) cr3 = __read_cr3();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) vmcs_writel(HOST_CR3, cr3); /* 22.2.3 FIXME: shadow tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) vmx->loaded_vmcs->host_state.cr3 = cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) /* Save the most likely value for this task's CR4 in the VMCS. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) cr4 = cr4_read_shadow();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) vmcs_writel(HOST_CR4, cr4); /* 22.2.3, 22.2.5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) vmx->loaded_vmcs->host_state.cr4 = cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) * Load null selectors, so we can avoid reloading them in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) * vmx_prepare_switch_to_host(), in case userspace uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) * the null selectors too (the expected case).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) vmcs_write16(HOST_DS_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) vmcs_write16(HOST_ES_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) vmcs_writel(HOST_IDTR_BASE, host_idt_base); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) vmcs_writel(HOST_RIP, (unsigned long)vmx_vmexit); /* 22.2.5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl); /* 22.2.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) rdmsr(MSR_IA32_CR_PAT, low32, high32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) if (cpu_has_load_ia32_efer())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) vmcs_write64(HOST_IA32_EFER, host_efer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) struct kvm_vcpu *vcpu = &vmx->vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) vcpu->arch.cr4_guest_owned_bits = KVM_POSSIBLE_CR4_GUEST_BITS &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) ~vcpu->arch.cr4_guest_rsvd_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) vcpu->arch.cr4_guest_owned_bits &= ~X86_CR4_PGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) if (is_guest_mode(&vmx->vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) vcpu->arch.cr4_guest_owned_bits &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) ~get_vmcs12(vcpu)->cr4_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) vmcs_writel(CR4_GUEST_HOST_MASK, ~vcpu->arch.cr4_guest_owned_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) if (!kvm_vcpu_apicv_active(&vmx->vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) if (!enable_vnmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) if (!enable_preemption_timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) return pin_based_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) if (cpu_has_secondary_exec_ctrls()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) if (kvm_vcpu_apicv_active(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) secondary_exec_controls_setbit(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) secondary_exec_controls_clearbit(vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) vmx_update_msr_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) u32 vmx_exec_control(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) exec_control &= ~CPU_BASED_MOV_DR_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) exec_control &= ~CPU_BASED_TPR_SHADOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) exec_control |= CPU_BASED_CR8_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) CPU_BASED_CR8_LOAD_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) exec_control |= CPU_BASED_CR3_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) CPU_BASED_INVLPG_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) if (kvm_mwait_in_guest(vmx->vcpu.kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) exec_control &= ~(CPU_BASED_MWAIT_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) CPU_BASED_MONITOR_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) if (kvm_hlt_in_guest(vmx->vcpu.kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) exec_control &= ~CPU_BASED_HLT_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) return exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) * Adjust a single secondary execution control bit to intercept/allow an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) * instruction in the guest. This is usually done based on whether or not a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) * feature has been exposed to the guest in order to correctly emulate faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) vmx_adjust_secondary_exec_control(struct vcpu_vmx *vmx, u32 *exec_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) u32 control, bool enabled, bool exiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) * If the control is for an opt-in feature, clear the control if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) * feature is not exposed to the guest, i.e. not enabled. If the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) * control is opt-out, i.e. an exiting control, clear the control if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) * the feature _is_ exposed to the guest, i.e. exiting/interception is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) * disabled for the associated instruction. Note, the caller is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) * responsible presetting exec_control to set all supported bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) if (enabled == exiting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) *exec_control &= ~control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) * Update the nested MSR settings so that a nested VMM can/can't set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) * controls for features that are/aren't exposed to the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) if (nested) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) if (enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) vmx->nested.msrs.secondary_ctls_high |= control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) vmx->nested.msrs.secondary_ctls_high &= ~control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) * Wrapper macro for the common case of adjusting a secondary execution control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) * based on a single guest CPUID bit, with a dedicated feature bit. This also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) * verifies that the control is actually supported by KVM and hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) #define vmx_adjust_sec_exec_control(vmx, exec_control, name, feat_name, ctrl_name, exiting) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) bool __enabled; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) if (cpu_has_vmx_##name()) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) __enabled = guest_cpuid_has(&(vmx)->vcpu, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) X86_FEATURE_##feat_name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) vmx_adjust_secondary_exec_control(vmx, exec_control, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) SECONDARY_EXEC_##ctrl_name, __enabled, exiting); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) /* More macro magic for ENABLE_/opt-in versus _EXITING/opt-out controls. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) #define vmx_adjust_sec_exec_feature(vmx, exec_control, lname, uname) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, ENABLE_##uname, false)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) #define vmx_adjust_sec_exec_exiting(vmx, exec_control, lname, uname) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) vmx_adjust_sec_exec_control(vmx, exec_control, lname, uname, uname##_EXITING, true)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) struct kvm_vcpu *vcpu = &vmx->vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) if (vmx_pt_mode_is_system())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) exec_control &= ~(SECONDARY_EXEC_PT_USE_GPA | SECONDARY_EXEC_PT_CONCEAL_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) if (!cpu_need_virtualize_apic_accesses(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) if (vmx->vpid == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) if (!enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) enable_unrestricted_guest = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) if (!enable_unrestricted_guest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) if (kvm_pause_in_guest(vmx->vcpu.kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) if (!kvm_vcpu_apicv_active(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) * in vmx_set_cr4. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) exec_control &= ~SECONDARY_EXEC_DESC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) (handle_vmptrld).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) We can NOT enable shadow_vmcs here because we don't have yet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) a current VMCS12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) if (!enable_pml)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) if (cpu_has_vmx_xsaves()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) /* Exposing XSAVES only when XSAVE is exposed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) bool xsaves_enabled =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) boot_cpu_has(X86_FEATURE_XSAVE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) vcpu->arch.xsaves_enabled = xsaves_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) vmx_adjust_secondary_exec_control(vmx, &exec_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) SECONDARY_EXEC_XSAVES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) xsaves_enabled, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) vmx_adjust_sec_exec_feature(vmx, &exec_control, rdtscp, RDTSCP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) * Expose INVPCID if and only if PCID is also exposed to the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) * INVPCID takes a #UD when it's disabled in the VMCS, but a #GP or #PF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) * if CR4.PCIDE=0. Enumerating CPUID.INVPCID=1 would lead to incorrect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) * behavior from the guest perspective (it would expect #GP or #PF).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) vmx_adjust_sec_exec_control(vmx, &exec_control, waitpkg, WAITPKG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) ENABLE_USR_WAIT_PAUSE, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) vmx->secondary_exec_control = exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) static void ept_set_mmio_spte_mask(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) * EPT Misconfigurations can be generated if the value of bits 2:0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) * of an EPT paging-structure entry is 110b (write/execute).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) kvm_mmu_set_mmio_spte_mask(VMX_EPT_MISCONFIG_WX_VALUE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) #define VMX_XSS_EXIT_BITMAP 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) * Noting that the initialization of Guest-state Area of VMCS is in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) * vmx_vcpu_reset().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) static void init_vmcs(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) if (nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) nested_vmx_set_vmcs_shadowing_bitmap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) /* Control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) pin_controls_set(vmx, vmx_pin_based_exec_ctrl(vmx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) exec_controls_set(vmx, vmx_exec_control(vmx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) if (cpu_has_secondary_exec_ctrls()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) vmx_compute_secondary_exec_control(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) secondary_exec_controls_set(vmx, vmx->secondary_exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) vmcs_write64(EOI_EXIT_BITMAP0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) vmcs_write64(EOI_EXIT_BITMAP1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) vmcs_write64(EOI_EXIT_BITMAP2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) vmcs_write64(EOI_EXIT_BITMAP3, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) vmcs_write16(GUEST_INTR_STATUS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) vmcs_write32(PLE_GAP, ple_gap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) vmx->ple_window = ple_window;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) vmx->ple_window_dirty = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) vmcs_write16(HOST_FS_SELECTOR, 0); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) vmcs_write16(HOST_GS_SELECTOR, 0); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) vmx_set_constant_host_state(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) if (cpu_has_vmx_vmfunc())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) vmcs_write64(VM_FUNCTION_CONTROL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) vm_exit_controls_set(vmx, vmx_vmexit_ctrl());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) /* 22.2.1, 20.8.1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) vm_entry_controls_set(vmx, vmx_vmentry_ctrl());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) vmx->vcpu.arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) set_cr4_guest_host_mask(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) if (vmx->vpid != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) if (cpu_has_vmx_xsaves())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) if (enable_pml) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) if (cpu_has_vmx_encls_vmexit())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) if (vmx_pt_mode_is_host_guest()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) /* Bit[6~0] are forced to 1, writes are ignored. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) vmx->pt_desc.guest.output_mask = 0x7F;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) struct msr_data apic_base_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) u64 cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) vmx->rmode.vm86_active = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) vmx->spec_ctrl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) vmx->msr_ia32_umwait_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) vmx->hv_deadline_tsc = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) kvm_set_cr8(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) if (!init_event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) MSR_IA32_APICBASE_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) if (kvm_vcpu_is_reset_bsp(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) apic_base_msr.host_initiated = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) kvm_set_apic_base(vcpu, &apic_base_msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) vmx_segment_cache_clear(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) seg_setup(VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) seg_setup(VCPU_SREG_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) seg_setup(VCPU_SREG_ES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) seg_setup(VCPU_SREG_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) seg_setup(VCPU_SREG_GS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) seg_setup(VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) vmcs_write16(GUEST_TR_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) vmcs_writel(GUEST_TR_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) vmcs_write32(GUEST_TR_LIMIT, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) vmcs_write16(GUEST_LDTR_SELECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) vmcs_writel(GUEST_LDTR_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) if (!init_event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) vmcs_write32(GUEST_SYSENTER_CS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) vmcs_writel(GUEST_SYSENTER_ESP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) vmcs_writel(GUEST_SYSENTER_EIP, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) kvm_rip_write(vcpu, 0xfff0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) vmcs_writel(GUEST_GDTR_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) vmcs_writel(GUEST_IDTR_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) if (kvm_mpx_supported())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) vmcs_write64(GUEST_BNDCFGS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) setup_msrs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) if (cpu_has_vmx_tpr_shadow() && !init_event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) if (cpu_need_tpr_shadow(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) __pa(vcpu->arch.apic->regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) vmcs_write32(TPR_THRESHOLD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) vmx->vcpu.arch.cr0 = cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) vmx_set_cr0(vcpu, cr0); /* enter rmode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) vmx_set_cr4(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) vmx_set_efer(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) update_exception_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) vpid_sync_context(vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) if (init_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) vmx_clear_hlt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) static void enable_irq_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) static void enable_nmi_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) if (!enable_vnmi ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) enable_irq_window(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) static void vmx_inject_irq(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) uint32_t intr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) int irq = vcpu->arch.interrupt.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) trace_kvm_inj_virq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) ++vcpu->stat.irq_injections;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) if (vmx->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) int inc_eip = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) if (vcpu->arch.interrupt.soft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) inc_eip = vcpu->arch.event_exit_inst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) kvm_inject_realmode_interrupt(vcpu, irq, inc_eip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) intr = irq | INTR_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) if (vcpu->arch.interrupt.soft) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) intr |= INTR_TYPE_SOFT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) vmx->vcpu.arch.event_exit_inst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) intr |= INTR_TYPE_EXT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) vmx_clear_hlt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) if (!enable_vnmi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) * Tracking the NMI-blocked state in software is built upon
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) * finding the next open IRQ window. This, in turn, depends on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) * well-behaving guests: They have to keep IRQs disabled at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) * least as long as the NMI handler runs. Otherwise we may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) * cause NMI nesting, maybe breaking the guest. But as this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) * highly unlikely, we can live with the residual risk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) vmx->loaded_vmcs->soft_vnmi_blocked = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) vmx->loaded_vmcs->vnmi_blocked_time = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) ++vcpu->stat.nmi_injections;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) vmx->loaded_vmcs->nmi_known_unmasked = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) if (vmx->rmode.vm86_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) vmx_clear_hlt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) bool masked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) if (!enable_vnmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) return vmx->loaded_vmcs->soft_vnmi_blocked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) if (vmx->loaded_vmcs->nmi_known_unmasked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) vmx->loaded_vmcs->nmi_known_unmasked = !masked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) return masked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) if (!enable_vnmi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) vmx->loaded_vmcs->soft_vnmi_blocked = masked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) vmx->loaded_vmcs->vnmi_blocked_time = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) vmx->loaded_vmcs->nmi_known_unmasked = !masked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) if (masked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) bool vmx_nmi_blocked(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) GUEST_INTR_STATE_NMI));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) if (to_vmx(vcpu)->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) return !vmx_nmi_blocked(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) if (to_vmx(vcpu)->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) * e.g. if the IRQ arrived asynchronously after checking nested events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) return !vmx_interrupt_blocked(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) if (enable_unrestricted_guest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) ret = __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) PAGE_SIZE * 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) to_kvm_vmx(kvm)->tss_addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) return init_rmode_tss(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) switch (vec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) case BP_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) * Update instruction length as we may reinject the exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) * from user space while in guest debugging mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) case DB_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) return !(vcpu->guest_debug &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) case DE_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) case OF_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) case BR_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) case UD_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) case DF_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) case SS_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) case GP_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) case MF_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) static int handle_rmode_exception(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) int vec, u32 err_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) * Instruction with address size override prefix opcode 0x67
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) * Cause the #SS fault with 0 error code in VM86 mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) if (kvm_emulate_instruction(vcpu, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) if (vcpu->arch.halt_request) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) vcpu->arch.halt_request = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) return kvm_vcpu_halt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) * Forward all other exceptions that are valid in real mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) * FIXME: Breaks guest debugging in real mode, needs to be fixed with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) * the required debugging infrastructure rework.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) kvm_queue_exception(vcpu, vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) * Trigger machine check on the host. We assume all the MSRs are already set up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) * by the CPU and that we still run on the same CPU as the MCE occurred on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) * We pass a fake environment to the machine check handler because we want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) * the guest to be always treated like user space, no matter what context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) * it used internally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) static void kvm_machine_check(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) #if defined(CONFIG_X86_MCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) struct pt_regs regs = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) .cs = 3, /* Fake ring 3 no matter what the guest ran on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) .flags = X86_EFLAGS_IF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) do_machine_check(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) static int handle_machine_check(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) /* handled by vmx_vcpu_run() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) * If the host has split lock detection disabled, then #AC is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) * unconditionally injected into the guest, which is the pre split lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) * detection behaviour.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) * If the host has split lock detection enabled then #AC is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) * only injected into the guest when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) * - Guest CPL == 3 (user mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) * - Guest has #AC detection enabled in CR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) * - Guest EFLAGS has AC bit set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) return vmx_get_cpl(vcpu) == 3 && kvm_read_cr0_bits(vcpu, X86_CR0_AM) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) static int handle_exception_nmi(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) struct kvm_run *kvm_run = vcpu->run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) u32 intr_info, ex_no, error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) unsigned long cr2, rip, dr6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) u32 vect_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) vect_info = vmx->idt_vectoring_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) if (is_machine_check(intr_info) || is_nmi(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) return 1; /* handled by handle_exception_nmi_irqoff() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) if (is_invalid_opcode(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) return handle_ud(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) error_code = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) WARN_ON_ONCE(!enable_vmware_backdoor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) * VMware backdoor emulation on #GP interception only handles
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) * IN{S}, OUT{S}, and RDPMC, none of which generate a non-zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) * error code on #GP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) if (error_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) * MMIO, it is better to report an internal error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) * See the comments in vmx_handle_exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) if ((vect_info & VECTORING_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816) vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) vcpu->run->internal.ndata = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) vcpu->run->internal.data[0] = vect_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) vcpu->run->internal.data[1] = intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) vcpu->run->internal.data[2] = error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) vcpu->run->internal.data[3] = vcpu->arch.last_vmentry_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) if (is_page_fault(intr_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) cr2 = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) if (enable_ept && !vcpu->arch.apf.host_apf_flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) * EPT will cause page fault only if we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) * detect illegal GPAs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) WARN_ON_ONCE(!allow_smaller_maxphyaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) ex_no = intr_info & INTR_INFO_VECTOR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841) if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) return handle_rmode_exception(vcpu, ex_no, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) switch (ex_no) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) case DB_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) dr6 = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) if (!(vcpu->guest_debug &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) * If the #DB was due to ICEBP, a.k.a. INT1, skip the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) * instruction. ICEBP generates a trap-like #DB, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) * despite its interception control being tied to #DB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) * is an instruction intercept, i.e. the VM-Exit occurs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) * on the ICEBP itself. Note, skipping ICEBP also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) * clears STI and MOVSS blocking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) * if single-step is enabled in RFLAGS and STI or MOVSS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) * blocking is active, as the CPU doesn't set the bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) * on VM-Exit due to #DB interception. VM-Entry has a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) * consistency check that a single-step #DB is pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) * in this scenario as the previous instruction cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) * have toggled RFLAGS.TF 0=>1 (because STI and POP/MOV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) * don't modify RFLAGS), therefore the one instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) * delay when activating single-step breakpoints must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) * have already expired. Note, the CPU sets/clears BS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) * as appropriate for all other VM-Exits types.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) if (is_icebp(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) WARN_ON(!skip_emulated_instruction(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) else if ((vmx_get_rflags(vcpu) & X86_EFLAGS_TF) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS) | DR6_BS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) case BP_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) * Update instruction length as we may reinject #BP from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) * user space while in guest debugging mode. Reading it for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) * #DB as well causes no harm, it is not used in that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) vmx->vcpu.arch.event_exit_inst_len =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) kvm_run->exit_reason = KVM_EXIT_DEBUG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892) rip = kvm_rip_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) kvm_run->debug.arch.exception = ex_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896) case AC_VECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) if (vmx_guest_inject_ac(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) * Handle split lock. Depending on detection mode this will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) * either warn and disable split lock detection for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) * task or force SIGBUS on it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) if (handle_guest_split_lock(kvm_rip_read(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) kvm_run->ex.exception = ex_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) kvm_run->ex.error_code = error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) static __always_inline int handle_external_interrupt(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) ++vcpu->stat.irq_exits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) static int handle_triple_fault(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) vcpu->mmio_needed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) static int handle_io(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) int size, in, string;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) unsigned port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) string = (exit_qualification & 16) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) ++vcpu->stat.io_exits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) if (string)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) return kvm_emulate_instruction(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) port = exit_qualification >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) size = (exit_qualification & 7) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) in = (exit_qualification & 8) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) return kvm_fast_pio(vcpu, size, port, in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957) * Patch in the VMCALL instruction:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) hypercall[0] = 0x0f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) hypercall[1] = 0x01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) hypercall[2] = 0xc1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) unsigned long orig_val = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) * We get here when L2 changed cr0 in a way that did not change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) * but did change L0 shadowed bits. So we first calculate the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) * effective cr0 value that L1 would like to write into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) * hardware. It consists of the L2-owned bits from the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) * value combined with the L1-owned bits from L1's guest_cr0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) val = (val & ~vmcs12->cr0_guest_host_mask) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) if (!nested_guest_cr0_valid(vcpu, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) if (kvm_set_cr0(vcpu, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) vmcs_writel(CR0_READ_SHADOW, orig_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) if (to_vmx(vcpu)->nested.vmxon &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) !nested_host_cr0_valid(vcpu, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) return kvm_set_cr0(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) unsigned long orig_val = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) /* analogously to handle_set_cr0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) val = (val & ~vmcs12->cr4_guest_host_mask) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) if (kvm_set_cr4(vcpu, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) vmcs_writel(CR4_READ_SHADOW, orig_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) return kvm_set_cr4(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) static int handle_desc(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) return kvm_emulate_instruction(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) static int handle_cr(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) unsigned long exit_qualification, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) int cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) int reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) cr = exit_qualification & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) reg = (exit_qualification >> 8) & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) switch ((exit_qualification >> 4) & 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) case 0: /* mov to cr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) val = kvm_register_readl(vcpu, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) trace_kvm_cr_write(cr, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) switch (cr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) err = handle_set_cr0(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) return kvm_complete_insn_gp(vcpu, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) WARN_ON_ONCE(enable_unrestricted_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042) err = kvm_set_cr3(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) return kvm_complete_insn_gp(vcpu, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) err = handle_set_cr4(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) return kvm_complete_insn_gp(vcpu, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) case 8: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) u8 cr8_prev = kvm_get_cr8(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) u8 cr8 = (u8)val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) err = kvm_set_cr8(vcpu, cr8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) ret = kvm_complete_insn_gp(vcpu, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) if (lapic_in_kernel(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) if (cr8_prev <= cr8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) * TODO: we might be squashing a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) * KVM_GUESTDBG_SINGLESTEP-triggered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) * KVM_EXIT_DEBUG here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) case 2: /* clts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) WARN_ONCE(1, "Guest should always own CR0.TS");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) case 1: /*mov from cr*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) switch (cr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) WARN_ON_ONCE(enable_unrestricted_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) val = kvm_read_cr3(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) kvm_register_write(vcpu, reg, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) trace_kvm_cr_read(cr, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080) val = kvm_get_cr8(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) kvm_register_write(vcpu, reg, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) trace_kvm_cr_read(cr, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) case 3: /* lmsw */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) kvm_lmsw(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) vcpu->run->exit_reason = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) (int)(exit_qualification >> 4) & 3, cr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) static int handle_dr(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) int dr, dr7, reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) /* First, if DR does not exist, trigger UD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110) if (!kvm_require_dr(vcpu, dr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) /* Do not handle if the CPL > 0, will trigger GP on re-entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) if (!kvm_require_cpl(vcpu, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) dr7 = vmcs_readl(GUEST_DR7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) if (dr7 & DR7_GD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) * As the vm-exit takes precedence over the debug trap, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) * need to emulate the latter, either for the host or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) * guest debugging itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) vcpu->run->debug.arch.dr6 = DR6_BD | DR6_RTM | DR6_FIXED_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) vcpu->run->debug.arch.dr7 = dr7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) vcpu->run->debug.arch.exception = DB_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) vcpu->run->exit_reason = KVM_EXIT_DEBUG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) if (vcpu->guest_debug == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) * No more DR vmexits; force a reload of the debug registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) * and reenter on this instruction. The next vmexit will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) * retrieve the full state of the debug registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) reg = DEBUG_REG_ACCESS_REG(exit_qualification);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) if (exit_qualification & TYPE_MOV_FROM_DR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) if (kvm_get_dr(vcpu, dr, &val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) kvm_register_write(vcpu, reg, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) get_debugreg(vcpu->arch.db[0], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) get_debugreg(vcpu->arch.db[1], 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) get_debugreg(vcpu->arch.db[2], 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) get_debugreg(vcpu->arch.db[3], 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) get_debugreg(vcpu->arch.dr6, 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) exec_controls_setbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) vmcs_writel(GUEST_DR7, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) kvm_apic_update_ppr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) static int handle_interrupt_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) ++vcpu->stat.irq_window_exits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) static int handle_vmcall(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) return kvm_emulate_hypercall(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) static int handle_invd(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) /* Treat an INVD instruction as a NOP and just skip it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) static int handle_invlpg(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211) kvm_mmu_invlpg(vcpu, exit_qualification);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) static int handle_rdpmc(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) err = kvm_rdpmc(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) return kvm_complete_insn_gp(vcpu, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) static int handle_wbinvd(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) return kvm_emulate_wbinvd(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) static int handle_xsetbv(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) u64 new_bv = kvm_read_edx_eax(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) u32 index = kvm_rcx_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) if (kvm_set_xcr(vcpu, index, new_bv) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) static int handle_apic_access(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240) if (likely(fasteoi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) int access_type, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244) access_type = exit_qualification & APIC_ACCESS_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) offset = exit_qualification & APIC_ACCESS_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) * Sane guest uses MOV to write EOI, with written value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) * not cared. So make a short-circuit here by avoiding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) * heavy instruction emulation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251) if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) (offset == APIC_EOI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) kvm_lapic_set_eoi(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) return kvm_emulate_instruction(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) int vector = exit_qualification & 0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) kvm_apic_set_eoi_accelerated(vcpu, vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) static int handle_apic_write(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) u32 offset = exit_qualification & 0xfff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) /* APIC-write VM exit is trap-like and thus no need to adjust IP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) kvm_apic_write_nodecode(vcpu, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) static int handle_task_switch(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) bool has_error_code = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) u32 error_code = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) u16 tss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) int reason, type, idt_v, idt_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289) idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) reason = (u32)exit_qualification >> 30;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) if (reason == TASK_SWITCH_GATE && idt_v) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) case INTR_TYPE_NMI_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) vcpu->arch.nmi_injected = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) vmx_set_nmi_mask(vcpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) case INTR_TYPE_EXT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) case INTR_TYPE_SOFT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) kvm_clear_interrupt_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) case INTR_TYPE_HARD_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) if (vmx->idt_vectoring_info &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) VECTORING_INFO_DELIVER_CODE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) has_error_code = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) vmcs_read32(IDT_VECTORING_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314) case INTR_TYPE_SOFT_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) kvm_clear_exception_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) tss_selector = exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) type != INTR_TYPE_EXT_INTR &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) type != INTR_TYPE_NMI_INTR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) WARN_ON(!skip_emulated_instruction(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) * TODO: What about debug traps on tss switch?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) * Are we supposed to inject them and update dr6?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) return kvm_task_switch(vcpu, tss_selector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) type == INTR_TYPE_SOFT_INTR ? idt_index : -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334) reason, has_error_code, error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) static int handle_ept_violation(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) gpa_t gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) u64 error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) * EPT violation happened while executing iret from NMI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) * "blocked by NMI" bit has to be set before next VM entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) * There are errata that may cause this bit to not be set:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) * AAK134, BY25.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) enable_vnmi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) (exit_qualification & INTR_INFO_UNBLOCK_NMI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357) trace_kvm_page_fault(gpa, exit_qualification);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) /* Is it a read fault? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) ? PFERR_USER_MASK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) /* Is it a write fault? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) ? PFERR_WRITE_MASK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) /* Is it a fetch fault? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) ? PFERR_FETCH_MASK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) /* ept page table entry is present? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) error_code |= (exit_qualification &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) EPT_VIOLATION_EXECUTABLE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) ? PFERR_PRESENT_MASK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) error_code |= (exit_qualification & 0x100) != 0 ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) vcpu->arch.exit_qualification = exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) * Check that the GPA doesn't exceed physical memory limits, as that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) * a guest page fault. We have to emulate the instruction here, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) * if the illegal address is that of a paging structure, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) * EPT_VIOLATION_ACC_WRITE bit is set. Alternatively, if supported we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384) * would also use advanced VM-exit information for EPT violations to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) * reconstruct the page fault error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387) if (unlikely(allow_smaller_maxphyaddr && kvm_vcpu_is_illegal_gpa(vcpu, gpa)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) return kvm_emulate_instruction(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) gpa_t gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) * A nested guest cannot optimize MMIO vmexits, because we have an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) * nGPA here instead of the required GPA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) if (!is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403) !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) trace_kvm_fast_mmio(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) static int handle_nmi_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) WARN_ON_ONCE(!enable_vnmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) ++vcpu->stat.nmi_window_exits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424) bool intr_window_requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) unsigned count = 130;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) intr_window_requested = exec_controls_get(vmx) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) CPU_BASED_INTR_WINDOW_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430) while (vmx->emulation_required && count-- != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) if (intr_window_requested && !vmx_interrupt_blocked(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432) return handle_interrupt_window(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) if (kvm_test_request(KVM_REQ_EVENT, vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) if (!kvm_emulate_instruction(vcpu, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) if (vmx->emulation_required && !vmx->rmode.vm86_active &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) vcpu->arch.exception.pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) vcpu->run->internal.suberror =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) KVM_INTERNAL_ERROR_EMULATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445) vcpu->run->internal.ndata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449) if (vcpu->arch.halt_request) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450) vcpu->arch.halt_request = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451) return kvm_vcpu_halt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) * Note, return 1 and not 0, vcpu_run() will invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) * xfer_to_guest_mode() which will create a proper return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457) * code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459) if (__xfer_to_guest_mode_work_pending())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466) static void grow_ple_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469) unsigned int old = vmx->ple_window;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) vmx->ple_window = __grow_ple_window(old, ple_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472) ple_window_grow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) ple_window_max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475) if (vmx->ple_window != old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) vmx->ple_window_dirty = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477) trace_kvm_ple_window_update(vcpu->vcpu_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) vmx->ple_window, old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482) static void shrink_ple_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) unsigned int old = vmx->ple_window;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487) vmx->ple_window = __shrink_ple_window(old, ple_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) ple_window_shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489) ple_window);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491) if (vmx->ple_window != old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) vmx->ple_window_dirty = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493) trace_kvm_ple_window_update(vcpu->vcpu_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494) vmx->ple_window, old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498) static void vmx_enable_tdp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500) kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501) enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503) 0ull, VMX_EPT_EXECUTABLE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504) cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) VMX_EPT_RWX_MASK, 0ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507) ept_set_mmio_spte_mask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511) * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512) * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514) static int handle_pause(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516) if (!kvm_pause_in_guest(vcpu->kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) grow_ple_window(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521) * VM-execution control is ignored if CPL > 0. OTOH, KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) * never set PAUSE_EXITING and just set PLE if supported,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523) * so the vcpu must be CPL=0 if it gets a PAUSE exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525) kvm_vcpu_on_spin(vcpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) static int handle_nop(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) static int handle_mwait(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536) printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) return handle_nop(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540) static int handle_invalid_op(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) static int handle_monitor_trap(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551) static int handle_monitor(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553) printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554) return handle_nop(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557) static int handle_invpcid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) u32 vmx_instruction_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) unsigned long type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563) u64 pcid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) u64 gla;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565) } operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573) type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575) if (type > 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) kvm_inject_gp(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580) /* According to the Intel instruction reference, the memory operand
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) * is read even if it isn't needed (e.g., for type==all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584) vmx_instruction_info, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) sizeof(operand), &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588) return kvm_handle_invpcid(vcpu, type, gva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) static int handle_pml_full(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) trace_kvm_pml_full(vcpu->vcpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) * PML buffer FULL happened while executing iret from NMI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) * "blocked by NMI" bit has to be set before next VM entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603) if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604) enable_vnmi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605) (exit_qualification & INTR_INFO_UNBLOCK_NMI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) * PML buffer already flushed at beginning of VMEXIT. Nothing to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611) * here.., and there's no userspace involvement needed for PML.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616) static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) if (!vmx->req_immediate_exit &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621) !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622) kvm_lapic_expired_hv_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) return EXIT_FASTPATH_REENTER_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629) static int handle_preemption_timer(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631) handle_fastpath_preemption_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) * When nested=0, all VMX instruction VM Exits filter here. The handlers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637) * are overwritten by nested_vmx_setup() when nested=1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) static int handle_vmx_instruction(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) static int handle_encls(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) * SGX virtualization is not yet supported. There is no software
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) * enable bit for SGX, so we have to trap ENCLS and inject a #UD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650) * to prevent the guest from executing ENCLS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657) * The exit handlers return 1 if the exit was handled fully and guest execution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658) * may resume. Otherwise they set the kvm_run parameter to indicate what needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659) * to be done to userspace and return 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661) static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662) [EXIT_REASON_EXCEPTION_NMI] = handle_exception_nmi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663) [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664) [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665) [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666) [EXIT_REASON_IO_INSTRUCTION] = handle_io,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667) [EXIT_REASON_CR_ACCESS] = handle_cr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) [EXIT_REASON_DR_ACCESS] = handle_dr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669) [EXIT_REASON_CPUID] = kvm_emulate_cpuid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670) [EXIT_REASON_MSR_READ] = kvm_emulate_rdmsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671) [EXIT_REASON_MSR_WRITE] = kvm_emulate_wrmsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672) [EXIT_REASON_INTERRUPT_WINDOW] = handle_interrupt_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) [EXIT_REASON_HLT] = kvm_emulate_halt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) [EXIT_REASON_INVD] = handle_invd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675) [EXIT_REASON_INVLPG] = handle_invlpg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676) [EXIT_REASON_RDPMC] = handle_rdpmc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677) [EXIT_REASON_VMCALL] = handle_vmcall,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678) [EXIT_REASON_VMCLEAR] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) [EXIT_REASON_VMLAUNCH] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680) [EXIT_REASON_VMPTRLD] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681) [EXIT_REASON_VMPTRST] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) [EXIT_REASON_VMREAD] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683) [EXIT_REASON_VMRESUME] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) [EXIT_REASON_VMWRITE] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685) [EXIT_REASON_VMOFF] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686) [EXIT_REASON_VMON] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689) [EXIT_REASON_APIC_WRITE] = handle_apic_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) [EXIT_REASON_EOI_INDUCED] = handle_apic_eoi_induced,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691) [EXIT_REASON_WBINVD] = handle_wbinvd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692) [EXIT_REASON_XSETBV] = handle_xsetbv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694) [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) [EXIT_REASON_GDTR_IDTR] = handle_desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696) [EXIT_REASON_LDTR_TR] = handle_desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697) [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) [EXIT_REASON_EPT_MISCONFIG] = handle_ept_misconfig,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699) [EXIT_REASON_PAUSE_INSTRUCTION] = handle_pause,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700) [EXIT_REASON_MWAIT_INSTRUCTION] = handle_mwait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701) [EXIT_REASON_MONITOR_TRAP_FLAG] = handle_monitor_trap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) [EXIT_REASON_MONITOR_INSTRUCTION] = handle_monitor,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703) [EXIT_REASON_INVEPT] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704) [EXIT_REASON_INVVPID] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) [EXIT_REASON_RDRAND] = handle_invalid_op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706) [EXIT_REASON_RDSEED] = handle_invalid_op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) [EXIT_REASON_PML_FULL] = handle_pml_full,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708) [EXIT_REASON_INVPCID] = handle_invpcid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) [EXIT_REASON_VMFUNC] = handle_vmx_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) [EXIT_REASON_ENCLS] = handle_encls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) static const int kvm_vmx_max_exit_handlers =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715) ARRAY_SIZE(kvm_vmx_exit_handlers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717) static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718) u32 *intr_info, u32 *error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) *info1 = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) if (!(vmx->exit_reason.failed_vmentry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724) *info2 = vmx->idt_vectoring_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) *intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) if (is_exception_with_error_code(*intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) *error_code = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731) *info2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) *intr_info = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733) *error_code = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737) static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) if (vmx->pml_pg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) __free_page(vmx->pml_pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) vmx->pml_pg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745) static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748) u64 *pml_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749) u16 pml_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751) pml_idx = vmcs_read16(GUEST_PML_INDEX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) /* Do nothing if PML buffer is empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754) if (pml_idx == (PML_ENTITY_NUM - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) /* PML index always points to next available PML buffer entity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) if (pml_idx >= PML_ENTITY_NUM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759) pml_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761) pml_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) pml_buf = page_address(vmx->pml_pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764) for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765) u64 gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) gpa = pml_buf[pml_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768) WARN_ON(gpa & (PAGE_SIZE - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769) kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772) /* reset PML index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777) * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778) * Called before reporting dirty_bitmap to userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780) static void kvm_flush_pml_buffers(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) * We only need to kick vcpu out of guest mode here, as PML buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786) * is flushed at beginning of all VMEXITs, and it's obvious that only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787) * vcpus running in guest are possible to have unflushed GPAs in PML
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788) * buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) kvm_for_each_vcpu(i, vcpu, kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791) kvm_vcpu_kick(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794) static void vmx_dump_sel(char *name, uint32_t sel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) name, vmcs_read16(sel),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798) vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) static void vmx_dump_dtsel(char *name, uint32_t limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) pr_err("%s limit=0x%08x, base=0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) name, vmcs_read32(limit),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) void dump_vmcs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812) u32 vmentry_ctl, vmexit_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813) u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814) unsigned long cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) if (!dump_invalid_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821) vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823) cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825) cr4 = vmcs_readl(GUEST_CR4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) secondary_exec_control = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827) if (cpu_has_secondary_exec_ctrls())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830) pr_err("*** Guest State ***\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832) vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) vmcs_readl(CR0_GUEST_HOST_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835) cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836) pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) if (cpu_has_vmx_ept()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) pr_err("PDPTR0 = 0x%016llx PDPTR1 = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839) vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840) pr_err("PDPTR2 = 0x%016llx PDPTR3 = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) pr_err("RSP = 0x%016lx RIP = 0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844) vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) pr_err("RFLAGS=0x%08lx DR7 = 0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847) pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) vmcs_readl(GUEST_SYSENTER_ESP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) vmx_dump_sel("CS: ", GUEST_CS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851) vmx_dump_sel("DS: ", GUEST_DS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) vmx_dump_sel("SS: ", GUEST_SS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853) vmx_dump_sel("ES: ", GUEST_ES_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854) vmx_dump_sel("FS: ", GUEST_FS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) vmx_dump_sel("GS: ", GUEST_GS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856) vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857) vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858) vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) vmx_dump_sel("TR: ", GUEST_TR_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860) if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861) (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) vmcs_read64(GUEST_IA32_EFER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864) vmcs_read64(GUEST_IA32_PAT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) pr_err("DebugCtl = 0x%016llx DebugExceptions = 0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866) vmcs_read64(GUEST_IA32_DEBUGCTL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867) vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) if (cpu_has_load_perf_global_ctrl() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870) pr_err("PerfGlobCtl = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871) vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874) pr_err("Interruptibility = %08x ActivityState = %08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876) vmcs_read32(GUEST_ACTIVITY_STATE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877) if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) pr_err("InterruptStatus = %04x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879) vmcs_read16(GUEST_INTR_STATUS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) pr_err("*** Host State ***\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882) pr_err("RIP = 0x%016lx RSP = 0x%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883) vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886) vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888) vmcs_read16(HOST_TR_SELECTOR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891) vmcs_readl(HOST_TR_BASE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892) pr_err("GDTBase=%016lx IDTBase=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894) pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895) vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) vmcs_readl(HOST_CR4));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898) vmcs_readl(HOST_IA32_SYSENTER_ESP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899) vmcs_read32(HOST_IA32_SYSENTER_CS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900) vmcs_readl(HOST_IA32_SYSENTER_EIP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) pr_err("EFER = 0x%016llx PAT = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903) vmcs_read64(HOST_IA32_EFER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) vmcs_read64(HOST_IA32_PAT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905) if (cpu_has_load_perf_global_ctrl() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906) vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) pr_err("PerfGlobCtl = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) pr_err("*** Control State ***\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912) pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913) pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915) vmcs_read32(EXCEPTION_BITMAP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917) vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918) pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919) vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921) vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922) pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) vmcs_read32(VM_EXIT_INTR_INFO),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924) vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925) vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) pr_err(" reason=%08x qualification=%016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) pr_err("IDTVectoring: info=%08x errcode=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) vmcs_read32(IDT_VECTORING_INFO_FIELD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930) vmcs_read32(IDT_VECTORING_ERROR_CODE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932) if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) pr_err("TSC Multiplier = 0x%016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934) vmcs_read64(TSC_MULTIPLIER));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) u16 status = vmcs_read16(GUEST_INTR_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938) pr_err("SVI|RVI = %02x|%02x ", status >> 8, status & 0xff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940) pr_cont("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5941) if (secondary_exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5942) pr_err("APIC-access addr = 0x%016llx ", vmcs_read64(APIC_ACCESS_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5943) pr_cont("virt-APIC addr = 0x%016llx\n", vmcs_read64(VIRTUAL_APIC_PAGE_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5945) if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5946) pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5947) if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5948) pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5949) if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5950) pr_err("PLE Gap=%08x Window=%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5951) vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5952) if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5953) pr_err("Virtual processor ID = 0x%04x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5954) vmcs_read16(VIRTUAL_PROCESSOR_ID));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5957) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5958) * The guest has exited. See if we can fix it or if we need userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5959) * assistance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5960) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5961) static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5963) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5964) union vmx_exit_reason exit_reason = vmx->exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5965) u32 vectoring_info = vmx->idt_vectoring_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5966) u16 exit_handler_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5968) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5969) * Flush logged GPAs PML buffer, this will make dirty_bitmap more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5970) * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5971) * querying dirty_bitmap, we only need to kick all vcpus out of guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5972) * mode as if vcpus is in root mode, the PML buffer must has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5973) * flushed already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5974) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5975) if (enable_pml)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5976) vmx_flush_pml_buffer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5978) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5979) * We should never reach this point with a pending nested VM-Enter, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5980) * more specifically emulation of L2 due to invalid guest state (see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5981) * below) should never happen as that means we incorrectly allowed a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5982) * nested VM-Enter with an invalid vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5983) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5984) WARN_ON_ONCE(vmx->nested.nested_run_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5986) /* If guest state is invalid, start emulating */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5987) if (vmx->emulation_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5988) return handle_invalid_guest_state(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5990) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5991) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5992) * The host physical addresses of some pages of guest memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5993) * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5994) * Page). The CPU may write to these pages via their host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5995) * physical address while L2 is running, bypassing any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5996) * address-translation-based dirty tracking (e.g. EPT write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5997) * protection).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5998) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5999) * Mark them dirty on every exit from L2 to prevent them from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6000) * getting out of sync with dirty tracking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6002) nested_mark_vmcs12_pages_dirty(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6004) if (nested_vmx_reflect_vmexit(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6005) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6008) if (exit_reason.failed_vmentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6009) dump_vmcs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6010) vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6011) vcpu->run->fail_entry.hardware_entry_failure_reason
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6012) = exit_reason.full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6013) vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6014) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6017) if (unlikely(vmx->fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6018) dump_vmcs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6019) vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6020) vcpu->run->fail_entry.hardware_entry_failure_reason
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6021) = vmcs_read32(VM_INSTRUCTION_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6022) vcpu->run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6023) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6026) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6027) * Note:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6028) * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6029) * delivery event since it indicates guest is accessing MMIO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6030) * The vm-exit can be triggered again after return to guest that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6031) * will cause infinite loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6032) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6033) if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6034) (exit_reason.basic != EXIT_REASON_EXCEPTION_NMI &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6035) exit_reason.basic != EXIT_REASON_EPT_VIOLATION &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6036) exit_reason.basic != EXIT_REASON_PML_FULL &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6037) exit_reason.basic != EXIT_REASON_APIC_ACCESS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6038) exit_reason.basic != EXIT_REASON_TASK_SWITCH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6039) int ndata = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6041) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6042) vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6043) vcpu->run->internal.data[0] = vectoring_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6044) vcpu->run->internal.data[1] = exit_reason.full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6045) vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6046) if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6047) vcpu->run->internal.data[ndata++] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6048) vmcs_read64(GUEST_PHYSICAL_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6050) vcpu->run->internal.data[ndata++] = vcpu->arch.last_vmentry_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6051) vcpu->run->internal.ndata = ndata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6052) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6055) if (unlikely(!enable_vnmi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6056) vmx->loaded_vmcs->soft_vnmi_blocked)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6057) if (!vmx_interrupt_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6058) vmx->loaded_vmcs->soft_vnmi_blocked = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6059) } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6060) vcpu->arch.nmi_pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6061) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6062) * This CPU don't support us in finding the end of an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6063) * NMI-blocked window if the guest runs with IRQs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6064) * disabled. So we pull the trigger after 1 s of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6065) * futile waiting, but inform the user about this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6066) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6067) printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6068) "state on VCPU %d after 1 s timeout\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6069) __func__, vcpu->vcpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6070) vmx->loaded_vmcs->soft_vnmi_blocked = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6074) if (exit_fastpath != EXIT_FASTPATH_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6075) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6077) if (exit_reason.basic >= kvm_vmx_max_exit_handlers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6078) goto unexpected_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6079) #ifdef CONFIG_RETPOLINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6080) if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6081) return kvm_emulate_wrmsr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6082) else if (exit_reason.basic == EXIT_REASON_PREEMPTION_TIMER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6083) return handle_preemption_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6084) else if (exit_reason.basic == EXIT_REASON_INTERRUPT_WINDOW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6085) return handle_interrupt_window(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6086) else if (exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6087) return handle_external_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6088) else if (exit_reason.basic == EXIT_REASON_HLT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6089) return kvm_emulate_halt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6090) else if (exit_reason.basic == EXIT_REASON_EPT_MISCONFIG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6091) return handle_ept_misconfig(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6092) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6094) exit_handler_index = array_index_nospec((u16)exit_reason.basic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6095) kvm_vmx_max_exit_handlers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6096) if (!kvm_vmx_exit_handlers[exit_handler_index])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6097) goto unexpected_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6099) return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6101) unexpected_vmexit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6102) vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6103) exit_reason.full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6104) dump_vmcs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6105) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6106) vcpu->run->internal.suberror =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6107) KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6108) vcpu->run->internal.ndata = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6109) vcpu->run->internal.data[0] = exit_reason.full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6110) vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6111) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6115) * Software based L1D cache flush which is used when microcode providing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6116) * the cache control MSR is not loaded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6117) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6118) * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6119) * flush it is required to read in 64 KiB because the replacement algorithm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6120) * is not exactly LRU. This could be sized at runtime via topology
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6121) * information but as all relevant affected CPUs have 32KiB L1D cache size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6122) * there is no point in doing so.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6124) static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6126) int size = PAGE_SIZE << L1D_CACHE_ORDER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6129) * This code is only executed when the the flush mode is 'cond' or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6130) * 'always'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6132) if (static_branch_likely(&vmx_l1d_flush_cond)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6133) bool flush_l1d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6135) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6136) * Clear the per-vcpu flush bit, it gets set again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6137) * either from vcpu_run() or from one of the unsafe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6138) * VMEXIT handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6140) flush_l1d = vcpu->arch.l1tf_flush_l1d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6141) vcpu->arch.l1tf_flush_l1d = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6144) * Clear the per-cpu flush bit, it gets set again from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6145) * the interrupt handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6147) flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6148) kvm_clear_cpu_l1tf_flush_l1d();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6150) if (!flush_l1d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6151) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6154) vcpu->stat.l1d_flush++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6156) if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6157) native_wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6158) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6161) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6162) /* First ensure the pages are in the TLB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6163) "xorl %%eax, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6164) ".Lpopulate_tlb:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6165) "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6166) "addl $4096, %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6167) "cmpl %%eax, %[size]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6168) "jne .Lpopulate_tlb\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6169) "xorl %%eax, %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6170) "cpuid\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6171) /* Now fill the cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6172) "xorl %%eax, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6173) ".Lfill_cache:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6174) "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6175) "addl $64, %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6176) "cmpl %%eax, %[size]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6177) "jne .Lfill_cache\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6178) "lfence\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6179) :: [flush_pages] "r" (vmx_l1d_flush_pages),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6180) [size] "r" (size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6181) : "eax", "ebx", "ecx", "edx");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6184) static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6186) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6187) int tpr_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6189) if (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6190) nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6191) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6193) tpr_threshold = (irr == -1 || tpr < irr) ? 0 : irr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6194) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6195) to_vmx(vcpu)->nested.l1_tpr_threshold = tpr_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6196) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6197) vmcs_write32(TPR_THRESHOLD, tpr_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6200) void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6202) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6203) u32 sec_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6205) if (!lapic_in_kernel(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6206) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6208) if (!flexpriority_enabled &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6209) !cpu_has_vmx_virtualize_x2apic_mode())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6210) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6212) /* Postpone execution until vmcs01 is the current VMCS. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6213) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6214) vmx->nested.change_vmcs01_virtual_apic_mode = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6215) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6218) sec_exec_control = secondary_exec_controls_get(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6219) sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6220) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6222) switch (kvm_get_apic_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6223) case LAPIC_MODE_INVALID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6224) WARN_ONCE(true, "Invalid local APIC state");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6225) case LAPIC_MODE_DISABLED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6226) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6227) case LAPIC_MODE_XAPIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6228) if (flexpriority_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6229) sec_exec_control |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6230) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6231) kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6234) * Flush the TLB, reloading the APIC access page will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6235) * only do so if its physical address has changed, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6236) * the guest may have inserted a non-APIC mapping into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6237) * the TLB while the APIC access page was disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6239) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6241) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6242) case LAPIC_MODE_X2APIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6243) if (cpu_has_vmx_virtualize_x2apic_mode())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6244) sec_exec_control |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6245) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6246) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6248) secondary_exec_controls_set(vmx, sec_exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6250) vmx_update_msr_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6253) static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6255) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6257) /* Defer reload until vmcs01 is the current VMCS. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6258) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6259) to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6260) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6263) if (!(secondary_exec_controls_get(to_vmx(vcpu)) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6264) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6265) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6267) page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6268) if (is_error_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6269) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6271) vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6272) vmx_flush_tlb_current(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6275) * Do not pin apic access page in memory, the MMU notifier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6276) * will call us again if it is migrated or swapped out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6278) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6281) static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6283) u16 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6284) u8 old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6286) if (max_isr == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6287) max_isr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6289) status = vmcs_read16(GUEST_INTR_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6290) old = status >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6291) if (max_isr != old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6292) status &= 0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6293) status |= max_isr << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6294) vmcs_write16(GUEST_INTR_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6298) static void vmx_set_rvi(int vector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6300) u16 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6301) u8 old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6303) if (vector == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6304) vector = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6306) status = vmcs_read16(GUEST_INTR_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6307) old = (u8)status & 0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6308) if ((u8)vector != old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6309) status &= ~0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6310) status |= (u8)vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6311) vmcs_write16(GUEST_INTR_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6315) static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6317) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6318) * When running L2, updating RVI is only relevant when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6319) * vmcs12 virtual-interrupt-delivery enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6320) * However, it can be enabled only when L1 also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6321) * intercepts external-interrupts and in that case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6322) * we should not update vmcs02 RVI but instead intercept
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6323) * interrupt. Therefore, do nothing when running L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6325) if (!is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6326) vmx_set_rvi(max_irr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6329) static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6331) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6332) int max_irr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6333) bool max_irr_updated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6335) WARN_ON(!vcpu->arch.apicv_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6336) if (pi_test_on(&vmx->pi_desc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6337) pi_clear_on(&vmx->pi_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6339) * IOMMU can write to PID.ON, so the barrier matters even on UP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6340) * But on x86 this is just a compiler barrier anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6342) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6343) max_irr_updated =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6344) kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6347) * If we are running L2 and L1 has a new pending interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6348) * which can be injected, this may cause a vmexit or it may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6349) * be injected into L2. Either way, this interrupt will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6350) * processed via KVM_REQ_EVENT, not RVI, because we do not use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6351) * virtual interrupt delivery to inject L1 interrupts into L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6352) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6353) if (is_guest_mode(vcpu) && max_irr_updated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6354) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6355) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6356) max_irr = kvm_lapic_find_highest_irr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6358) vmx_hwapic_irr_update(vcpu, max_irr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6359) return max_irr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6362) static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6364) if (!kvm_vcpu_apicv_active(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6365) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6367) vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6368) vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6369) vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6370) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6373) static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6375) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6377) pi_clear_on(&vmx->pi_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6378) memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6381) void vmx_do_interrupt_nmi_irqoff(unsigned long entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6383) static void handle_interrupt_nmi_irqoff(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6384) unsigned long entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6386) kvm_before_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6387) vmx_do_interrupt_nmi_irqoff(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6388) kvm_after_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6391) static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6393) const unsigned long nmi_entry = (unsigned long)asm_exc_nmi_noist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6394) u32 intr_info = vmx_get_intr_info(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6396) /* if exit due to PF check for async PF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6397) if (is_page_fault(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6398) vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6399) /* Handle machine checks before interrupts are enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6400) else if (is_machine_check(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6401) kvm_machine_check();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6402) /* We need to handle NMIs before interrupts are enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6403) else if (is_nmi(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6404) handle_interrupt_nmi_irqoff(&vmx->vcpu, nmi_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6407) static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6409) u32 intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6410) unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6411) gate_desc *desc = (gate_desc *)host_idt_base + vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6413) if (WARN_ONCE(!is_external_intr(intr_info),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6414) "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6415) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6417) handle_interrupt_nmi_irqoff(vcpu, gate_offset(desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6420) static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6422) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6424) if (vmx->emulation_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6425) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6427) if (vmx->exit_reason.basic == EXIT_REASON_EXTERNAL_INTERRUPT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6428) handle_external_interrupt_irqoff(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6429) else if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6430) handle_exception_nmi_irqoff(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6433) static bool vmx_has_emulated_msr(u32 index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6435) switch (index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6436) case MSR_IA32_SMBASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6438) * We cannot do SMM unless we can run the guest in big
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6439) * real mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6440) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6441) return enable_unrestricted_guest || emulate_invalid_guest_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6442) case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6443) return nested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6444) case MSR_AMD64_VIRT_SPEC_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6445) /* This is AMD only. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6446) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6447) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6448) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6452) static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6454) u32 exit_intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6455) bool unblock_nmi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6456) u8 vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6457) bool idtv_info_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6459) idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6461) if (enable_vnmi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6462) if (vmx->loaded_vmcs->nmi_known_unmasked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6463) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6465) exit_intr_info = vmx_get_intr_info(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6466) unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6467) vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6468) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6469) * SDM 3: 27.7.1.2 (September 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6470) * Re-set bit "block by NMI" before VM entry if vmexit caused by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6471) * a guest IRET fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6472) * SDM 3: 23.2.2 (September 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6473) * Bit 12 is undefined in any of the following cases:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6474) * If the VM exit sets the valid bit in the IDT-vectoring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6475) * information field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6476) * If the VM exit is due to a double fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6477) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6478) if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6479) vector != DF_VECTOR && !idtv_info_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6480) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6481) GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6482) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6483) vmx->loaded_vmcs->nmi_known_unmasked =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6484) !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6485) & GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6486) } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6487) vmx->loaded_vmcs->vnmi_blocked_time +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6488) ktime_to_ns(ktime_sub(ktime_get(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6489) vmx->loaded_vmcs->entry_time));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6492) static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6493) u32 idt_vectoring_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6494) int instr_len_field,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6495) int error_code_field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6497) u8 vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6498) int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6499) bool idtv_info_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6501) idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6503) vcpu->arch.nmi_injected = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6504) kvm_clear_exception_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6505) kvm_clear_interrupt_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6507) if (!idtv_info_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6508) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6510) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6512) vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6513) type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6515) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6516) case INTR_TYPE_NMI_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6517) vcpu->arch.nmi_injected = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6518) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6519) * SDM 3: 27.7.1.2 (September 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6520) * Clear bit "block by NMI" before VM entry if a NMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6521) * delivery faulted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6523) vmx_set_nmi_mask(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6524) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6525) case INTR_TYPE_SOFT_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6526) vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6527) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6528) case INTR_TYPE_HARD_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6529) if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6530) u32 err = vmcs_read32(error_code_field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6531) kvm_requeue_exception_e(vcpu, vector, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6532) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6533) kvm_requeue_exception(vcpu, vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6535) case INTR_TYPE_SOFT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6536) vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6537) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6538) case INTR_TYPE_EXT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6539) kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6540) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6541) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6542) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6546) static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6548) __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6549) VM_EXIT_INSTRUCTION_LEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6550) IDT_VECTORING_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6553) static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6555) __vmx_complete_interrupts(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6556) vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6557) VM_ENTRY_INSTRUCTION_LEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6558) VM_ENTRY_EXCEPTION_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6560) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6563) static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6565) int i, nr_msrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6566) struct perf_guest_switch_msr *msrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6568) msrs = perf_guest_get_msrs(&nr_msrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6570) if (!msrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6571) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6573) for (i = 0; i < nr_msrs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6574) if (msrs[i].host == msrs[i].guest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6575) clear_atomic_switch_msr(vmx, msrs[i].msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6576) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6577) add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6578) msrs[i].host, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6581) static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6583) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6584) u64 tscl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6585) u32 delta_tsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6587) if (vmx->req_immediate_exit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6588) vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6589) vmx->loaded_vmcs->hv_timer_soft_disabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6590) } else if (vmx->hv_deadline_tsc != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6591) tscl = rdtsc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6592) if (vmx->hv_deadline_tsc > tscl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6593) /* set_hv_timer ensures the delta fits in 32-bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6594) delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6595) cpu_preemption_timer_multi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6596) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6597) delta_tsc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6599) vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6600) vmx->loaded_vmcs->hv_timer_soft_disabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6601) } else if (!vmx->loaded_vmcs->hv_timer_soft_disabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6602) vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6603) vmx->loaded_vmcs->hv_timer_soft_disabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6607) void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6609) if (unlikely(host_rsp != vmx->loaded_vmcs->host_state.rsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6610) vmx->loaded_vmcs->host_state.rsp = host_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6611) vmcs_writel(HOST_RSP, host_rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6615) static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6616) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6617) switch (to_vmx(vcpu)->exit_reason.basic) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6618) case EXIT_REASON_MSR_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6619) return handle_fastpath_set_msr_irqoff(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6620) case EXIT_REASON_PREEMPTION_TIMER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6621) return handle_fastpath_preemption_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6622) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6623) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6627) bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6629) static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6630) struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6632) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6633) * VMENTER enables interrupts (host state), but the kernel state is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6634) * interrupts disabled when this is invoked. Also tell RCU about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6635) * it. This is the same logic as for exit_to_user_mode().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6636) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6637) * This ensures that e.g. latency analysis on the host observes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6638) * guest mode as interrupt enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6639) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6640) * guest_enter_irqoff() informs context tracking about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6641) * transition to guest mode and if enabled adjusts RCU state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6642) * accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6643) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6644) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6645) trace_hardirqs_on_prepare();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6646) lockdep_hardirqs_on_prepare(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6647) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6649) guest_enter_irqoff();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6650) lockdep_hardirqs_on(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6652) /* L1D Flush includes CPU buffer clear to mitigate MDS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6653) if (static_branch_unlikely(&vmx_l1d_should_flush))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6654) vmx_l1d_flush(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6655) else if (static_branch_unlikely(&mds_user_clear))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6656) mds_clear_cpu_buffers();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6658) if (vcpu->arch.cr2 != native_read_cr2())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6659) native_write_cr2(vcpu->arch.cr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6661) vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6662) vmx->loaded_vmcs->launched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6664) vcpu->arch.cr2 = native_read_cr2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6667) * VMEXIT disables interrupts (host state), but tracing and lockdep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6668) * have them in state 'on' as recorded before entering guest mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6669) * Same as enter_from_user_mode().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6670) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6671) * context_tracking_guest_exit() restores host context and reinstates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6672) * RCU if enabled and required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6673) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6674) * This needs to be done before the below as native_read_msr()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6675) * contains a tracepoint and x86_spec_ctrl_restore_host() calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6676) * into world and some more.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6677) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6678) lockdep_hardirqs_off(CALLER_ADDR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6679) context_tracking_guest_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6681) instrumentation_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6682) trace_hardirqs_off_finish();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6683) instrumentation_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6686) static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6688) fastpath_t exit_fastpath;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6689) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6690) unsigned long cr3, cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6692) reenter_guest:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6693) /* Record the guest's net vcpu time for enforced NMI injections. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6694) if (unlikely(!enable_vnmi &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6695) vmx->loaded_vmcs->soft_vnmi_blocked))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6696) vmx->loaded_vmcs->entry_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6698) /* Don't enter VMX if guest state is invalid, let the exit handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6699) start emulation until we arrive back to a valid state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6700) if (vmx->emulation_required)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6701) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6703) if (vmx->ple_window_dirty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6704) vmx->ple_window_dirty = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6705) vmcs_write32(PLE_WINDOW, vmx->ple_window);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6708) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6709) * We did this in prepare_switch_to_guest, because it needs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6710) * be within srcu_read_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6711) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6712) WARN_ON_ONCE(vmx->nested.need_vmcs12_to_shadow_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6714) if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6715) vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6716) if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6717) vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6719) cr3 = __get_current_cr3_fast();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6720) if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6721) vmcs_writel(HOST_CR3, cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6722) vmx->loaded_vmcs->host_state.cr3 = cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6725) cr4 = cr4_read_shadow();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6726) if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6727) vmcs_writel(HOST_CR4, cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6728) vmx->loaded_vmcs->host_state.cr4 = cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6731) /* When single-stepping over STI and MOV SS, we must clear the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6732) * corresponding interruptibility bits in the guest state. Otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6733) * vmentry fails as it then expects bit 14 (BS) in pending debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6734) * exceptions being set, but that's not correct for the guest debugging
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6735) * case. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6736) if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6737) vmx_set_interrupt_shadow(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6739) kvm_load_guest_xsave_state(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6741) pt_guest_enter(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6743) atomic_switch_perf_msrs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6745) if (enable_preemption_timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6746) vmx_update_hv_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6748) kvm_wait_lapic_expire(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6751) * If this vCPU has touched SPEC_CTRL, restore the guest's value if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6752) * it's non-zero. Since vmentry is serialising on affected CPUs, there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6753) * is no need to worry about the conditional branch over the wrmsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6754) * being speculatively taken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6755) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6756) x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6758) /* The actual VMENTER/EXIT is in the .noinstr.text section. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6759) vmx_vcpu_enter_exit(vcpu, vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6762) * We do not use IBRS in the kernel. If this vCPU has used the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6763) * SPEC_CTRL MSR it may have left it on; save the value and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6764) * turn it off. This is much more efficient than blindly adding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6765) * it to the atomic save/restore list. Especially as the former
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6766) * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6767) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6768) * For non-nested case:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6769) * If the L01 MSR bitmap does not intercept the MSR, then we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6770) * save it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6771) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6772) * For nested case:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6773) * If the L02 MSR bitmap does not intercept the MSR, then we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6774) * save it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6775) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6776) if (unlikely(!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6777) vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6779) x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6781) /* All fields are clean at this point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6782) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6783) current_evmcs->hv_clean_fields |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6784) HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6786) if (static_branch_unlikely(&enable_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6787) current_evmcs->hv_vp_id = vcpu->arch.hyperv.vp_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6789) /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6790) if (vmx->host_debugctlmsr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6791) update_debugctlmsr(vmx->host_debugctlmsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6793) #ifndef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6795) * The sysexit path does not restore ds/es, so we must set them to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6796) * a reasonable value ourselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6797) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6798) * We can't defer this to vmx_prepare_switch_to_host() since that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6799) * function may be executed in interrupt context, which saves and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6800) * restore segments around it, nullifying its effect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6802) loadsegment(ds, __USER_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6803) loadsegment(es, __USER_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6804) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6806) vmx_register_cache_reset(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6808) pt_guest_exit(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6810) kvm_load_host_xsave_state(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6812) vmx->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6813) vmx->idt_vectoring_info = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6815) if (unlikely(vmx->fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6816) vmx->exit_reason.full = 0xdead;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6817) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6820) vmx->exit_reason.full = vmcs_read32(VM_EXIT_REASON);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6821) if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6822) kvm_machine_check();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6824) trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6826) if (unlikely(vmx->exit_reason.failed_vmentry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6827) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6829) vmx->loaded_vmcs->launched = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6830) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6832) vmx_recover_nmi_blocking(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6833) vmx_complete_interrupts(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6835) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6836) return EXIT_FASTPATH_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6838) exit_fastpath = vmx_exit_handlers_fastpath(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6839) if (exit_fastpath == EXIT_FASTPATH_REENTER_GUEST) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6840) if (!kvm_vcpu_exit_request(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6841) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6842) * FIXME: this goto should be a loop in vcpu_enter_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6843) * but it would incur the cost of a retpoline for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6844) * Revisit once static calls are available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6846) if (vcpu->arch.apicv_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6847) vmx_sync_pir_to_irr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6848) goto reenter_guest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6850) exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6853) return exit_fastpath;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6856) static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6858) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6860) if (enable_pml)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6861) vmx_destroy_pml_buffer(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6862) free_vpid(vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6863) nested_vmx_free_vcpu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6864) free_loaded_vmcs(vmx->loaded_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6867) static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6868) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6869) struct vcpu_vmx *vmx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6870) int i, cpu, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6872) BUILD_BUG_ON(offsetof(struct vcpu_vmx, vcpu) != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6873) vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6875) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6877) vmx->vpid = allocate_vpid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6879) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6880) * If PML is turned on, failure on enabling PML just results in failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6881) * of creating the vcpu, therefore we can simplify PML logic (by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6882) * avoiding dealing with cases, such as enabling PML partially on vcpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6883) * for the guest), etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6884) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6885) if (enable_pml) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6886) vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6887) if (!vmx->pml_pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6888) goto free_vpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6891) BUILD_BUG_ON(ARRAY_SIZE(vmx_uret_msrs_list) != MAX_NR_USER_RETURN_MSRS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6893) for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6894) u32 index = vmx_uret_msrs_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6895) int j = vmx->nr_uret_msrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6897) if (kvm_probe_user_return_msr(index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6898) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6900) vmx->guest_uret_msrs[j].slot = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6901) vmx->guest_uret_msrs[j].data = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6902) switch (index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6903) case MSR_IA32_TSX_CTRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6905) * TSX_CTRL_CPUID_CLEAR is handled in the CPUID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6906) * interception. Keep the host value unchanged to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6907) * changing CPUID bits under the host kernel's feet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6908) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6909) * hle=0, rtm=0, tsx_ctrl=1 can be found with some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6910) * combinations of new kernel and old userspace. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6911) * those guests run on a tsx=off host, do allow guests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6912) * to use TSX_CTRL, but do not change the value on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6913) * host so that TSX remains always disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6914) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6915) if (boot_cpu_has(X86_FEATURE_RTM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6916) vmx->guest_uret_msrs[j].mask = ~(u64)TSX_CTRL_CPUID_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6917) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6918) vmx->guest_uret_msrs[j].mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6919) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6920) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6921) vmx->guest_uret_msrs[j].mask = -1ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6922) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6924) ++vmx->nr_uret_msrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6927) err = alloc_loaded_vmcs(&vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6928) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6929) goto free_pml;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6931) /* The MSR bitmap starts with all ones */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6932) bitmap_fill(vmx->shadow_msr_intercept.read, MAX_POSSIBLE_PASSTHROUGH_MSRS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6933) bitmap_fill(vmx->shadow_msr_intercept.write, MAX_POSSIBLE_PASSTHROUGH_MSRS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6935) vmx_disable_intercept_for_msr(vcpu, MSR_IA32_TSC, MSR_TYPE_R);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6936) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6937) vmx_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6938) vmx_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6939) vmx_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6940) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6941) vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6942) vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6943) vmx_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6944) if (kvm_cstate_in_guest(vcpu->kvm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6945) vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C1_RES, MSR_TYPE_R);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6946) vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C3_RESIDENCY, MSR_TYPE_R);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6947) vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C6_RESIDENCY, MSR_TYPE_R);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6948) vmx_disable_intercept_for_msr(vcpu, MSR_CORE_C7_RESIDENCY, MSR_TYPE_R);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6950) vmx->msr_bitmap_mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6952) vmx->loaded_vmcs = &vmx->vmcs01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6953) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6954) vmx_vcpu_load(vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6955) vcpu->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6956) init_vmcs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6957) vmx_vcpu_put(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6958) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6959) if (cpu_need_virtualize_apic_accesses(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6960) err = alloc_apic_access_page(vcpu->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6961) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6962) goto free_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6965) if (enable_ept && !enable_unrestricted_guest) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6966) err = init_rmode_identity_map(vcpu->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6967) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6968) goto free_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6971) if (nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6972) memcpy(&vmx->nested.msrs, &vmcs_config.nested, sizeof(vmx->nested.msrs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6973) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6974) memset(&vmx->nested.msrs, 0, sizeof(vmx->nested.msrs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6976) vmx->nested.posted_intr_nv = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6977) vmx->nested.current_vmptr = -1ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6979) vcpu->arch.microcode_version = 0x100000000ULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6980) vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6982) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6983) * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6984) * or POSTED_INTR_WAKEUP_VECTOR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6985) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6986) vmx->pi_desc.nv = POSTED_INTR_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6987) vmx->pi_desc.sn = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6989) vmx->ept_pointer = INVALID_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6991) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6993) free_vmcs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6994) free_loaded_vmcs(vmx->loaded_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6995) free_pml:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6996) vmx_destroy_pml_buffer(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6997) free_vpid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6998) free_vpid(vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6999) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7002) #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7003) #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7005) static int vmx_vm_init(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7006) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7007) spin_lock_init(&to_kvm_vmx(kvm)->ept_pointer_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7009) if (!ple_gap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7010) kvm->arch.pause_in_guest = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7012) if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7013) switch (l1tf_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7014) case L1TF_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7015) case L1TF_MITIGATION_FLUSH_NOWARN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7016) /* 'I explicitly don't care' is set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7017) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7018) case L1TF_MITIGATION_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7019) case L1TF_MITIGATION_FLUSH_NOSMT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7020) case L1TF_MITIGATION_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7021) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7022) * Warn upon starting the first VM in a potentially
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7023) * insecure environment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7024) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7025) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7026) pr_warn_once(L1TF_MSG_SMT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7027) if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7028) pr_warn_once(L1TF_MSG_L1D);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7029) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7030) case L1TF_MITIGATION_FULL_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7031) /* Flush is enforced */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7032) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7035) kvm_apicv_init(kvm, enable_apicv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7036) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7039) static int __init vmx_check_processor_compat(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7040) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7041) struct vmcs_config vmcs_conf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7042) struct vmx_capability vmx_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7044) if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7045) !this_cpu_has(X86_FEATURE_VMX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7046) pr_err("kvm: VMX is disabled on CPU %d\n", smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7047) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7050) if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7051) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7052) if (nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7053) nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7054) if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7055) printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7056) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7057) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7059) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7062) static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7063) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7064) u8 cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7065) u64 ipat = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7067) /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7068) * memory aliases with conflicting memory types and sometimes MCEs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7069) * We have to be careful as to what are honored and when.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7070) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7071) * For MMIO, guest CD/MTRR are ignored. The EPT memory type is set to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7072) * UC. The effective memory type is UC or WC depending on guest PAT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7073) * This was historically the source of MCEs and we want to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7074) * conservative.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7076) * When there is no need to deal with noncoherent DMA (e.g., no VT-d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7077) * or VT-d has snoop control), guest CD/MTRR/PAT are all ignored. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7078) * EPT memory type is set to WB. The effective memory type is forced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7079) * WB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7080) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7081) * Otherwise, we trust guest. Guest CD/MTRR/PAT are all honored. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7082) * EPT memory type is used to emulate guest CD/MTRR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7083) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7085) if (is_mmio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7086) cache = MTRR_TYPE_UNCACHABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7087) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7090) if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7091) ipat = VMX_EPT_IPAT_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7092) cache = MTRR_TYPE_WRBACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7093) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7096) if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7097) ipat = VMX_EPT_IPAT_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7098) if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7099) cache = MTRR_TYPE_WRBACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7100) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7101) cache = MTRR_TYPE_UNCACHABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7102) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7105) cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7107) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7108) return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7111) static void vmcs_set_secondary_exec_control(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7114) * These bits in the secondary execution controls field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7115) * are dynamic, the others are mostly based on the hypervisor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7116) * architecture and the guest's CPUID. Do not touch the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7117) * dynamic bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7119) u32 mask =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7120) SECONDARY_EXEC_SHADOW_VMCS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7121) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7122) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7123) SECONDARY_EXEC_DESC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7125) u32 new_ctl = vmx->secondary_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7126) u32 cur_ctl = secondary_exec_controls_get(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7128) secondary_exec_controls_set(vmx, (new_ctl & ~mask) | (cur_ctl & mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7132) * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7133) * (indicating "allowed-1") if they are supported in the guest's CPUID.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7135) static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7137) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7138) struct kvm_cpuid_entry2 *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7140) vmx->nested.msrs.cr0_fixed1 = 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7141) vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7143) #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7144) if (entry && (entry->_reg & (_cpuid_mask))) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7145) vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7146) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7148) entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7149) cr4_fixed1_update(X86_CR4_VME, edx, feature_bit(VME));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7150) cr4_fixed1_update(X86_CR4_PVI, edx, feature_bit(VME));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7151) cr4_fixed1_update(X86_CR4_TSD, edx, feature_bit(TSC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7152) cr4_fixed1_update(X86_CR4_DE, edx, feature_bit(DE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7153) cr4_fixed1_update(X86_CR4_PSE, edx, feature_bit(PSE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7154) cr4_fixed1_update(X86_CR4_PAE, edx, feature_bit(PAE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7155) cr4_fixed1_update(X86_CR4_MCE, edx, feature_bit(MCE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7156) cr4_fixed1_update(X86_CR4_PGE, edx, feature_bit(PGE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7157) cr4_fixed1_update(X86_CR4_OSFXSR, edx, feature_bit(FXSR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7158) cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, feature_bit(XMM));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7159) cr4_fixed1_update(X86_CR4_VMXE, ecx, feature_bit(VMX));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7160) cr4_fixed1_update(X86_CR4_SMXE, ecx, feature_bit(SMX));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7161) cr4_fixed1_update(X86_CR4_PCIDE, ecx, feature_bit(PCID));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7162) cr4_fixed1_update(X86_CR4_OSXSAVE, ecx, feature_bit(XSAVE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7164) entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7165) cr4_fixed1_update(X86_CR4_FSGSBASE, ebx, feature_bit(FSGSBASE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7166) cr4_fixed1_update(X86_CR4_SMEP, ebx, feature_bit(SMEP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7167) cr4_fixed1_update(X86_CR4_SMAP, ebx, feature_bit(SMAP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7168) cr4_fixed1_update(X86_CR4_PKE, ecx, feature_bit(PKU));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7169) cr4_fixed1_update(X86_CR4_UMIP, ecx, feature_bit(UMIP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7170) cr4_fixed1_update(X86_CR4_LA57, ecx, feature_bit(LA57));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7172) #undef cr4_fixed1_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7175) static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7177) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7179) if (kvm_mpx_supported()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7180) bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7182) if (mpx_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7183) vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7184) vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7185) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7186) vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7187) vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7192) static void update_intel_pt_cfg(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7194) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7195) struct kvm_cpuid_entry2 *best = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7196) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7198) for (i = 0; i < PT_CPUID_LEAVES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7199) best = kvm_find_cpuid_entry(vcpu, 0x14, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7200) if (!best)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7201) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7202) vmx->pt_desc.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM] = best->eax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7203) vmx->pt_desc.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM] = best->ebx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7204) vmx->pt_desc.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM] = best->ecx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7205) vmx->pt_desc.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM] = best->edx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7208) /* Get the number of configurable Address Ranges for filtering */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7209) vmx->pt_desc.addr_range = intel_pt_validate_cap(vmx->pt_desc.caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7210) PT_CAP_num_address_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7212) /* Initialize and clear the no dependency bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7213) vmx->pt_desc.ctl_bitmask = ~(RTIT_CTL_TRACEEN | RTIT_CTL_OS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7214) RTIT_CTL_USR | RTIT_CTL_TSC_EN | RTIT_CTL_DISRETC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7217) * If CPUID.(EAX=14H,ECX=0):EBX[0]=1 CR3Filter can be set otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7218) * will inject an #GP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7220) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_cr3_filtering))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7221) vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_CR3EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7224) * If CPUID.(EAX=14H,ECX=0):EBX[1]=1 CYCEn, CycThresh and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7225) * PSBFreq can be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7227) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7228) vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_CYCLEACC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7229) RTIT_CTL_CYC_THRESH | RTIT_CTL_PSB_FREQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7231) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7232) * If CPUID.(EAX=14H,ECX=0):EBX[3]=1 MTCEn BranchEn and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7233) * MTCFreq can be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7234) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7235) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7236) vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_MTC_EN |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7237) RTIT_CTL_BRANCH_EN | RTIT_CTL_MTC_RANGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7239) /* If CPUID.(EAX=14H,ECX=0):EBX[4]=1 FUPonPTW and PTWEn can be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7240) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_ptwrite))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7241) vmx->pt_desc.ctl_bitmask &= ~(RTIT_CTL_FUP_ON_PTW |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7242) RTIT_CTL_PTW_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7244) /* If CPUID.(EAX=14H,ECX=0):EBX[5]=1 PwrEvEn can be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7245) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_power_event_trace))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7246) vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_PWR_EVT_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7248) /* If CPUID.(EAX=14H,ECX=0):ECX[0]=1 ToPA can be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7249) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_topa_output))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7250) vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_TOPA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7252) /* If CPUID.(EAX=14H,ECX=0):ECX[3]=1 FabircEn can be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7253) if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_output_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7254) vmx->pt_desc.ctl_bitmask &= ~RTIT_CTL_FABRIC_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7256) /* unmask address range configure area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7257) for (i = 0; i < vmx->pt_desc.addr_range; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7258) vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7261) static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7263) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7265) /* xsaves_enabled is recomputed in vmx_compute_secondary_exec_control(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7266) vcpu->arch.xsaves_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7268) if (cpu_has_secondary_exec_ctrls()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7269) vmx_compute_secondary_exec_control(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7270) vmcs_set_secondary_exec_control(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7273) if (nested_vmx_allowed(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7274) to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7275) FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7276) FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7277) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7278) to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7279) ~(FEAT_CTL_VMX_ENABLED_INSIDE_SMX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7280) FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7282) if (nested_vmx_allowed(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7283) nested_vmx_cr_fixed1_bits_update(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7284) nested_vmx_entry_exit_ctls_update(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7287) if (boot_cpu_has(X86_FEATURE_INTEL_PT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7288) guest_cpuid_has(vcpu, X86_FEATURE_INTEL_PT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7289) update_intel_pt_cfg(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7291) if (boot_cpu_has(X86_FEATURE_RTM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7292) struct vmx_uret_msr *msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7293) msr = vmx_find_uret_msr(vmx, MSR_IA32_TSX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7294) if (msr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7295) bool enabled = guest_cpuid_has(vcpu, X86_FEATURE_RTM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7296) vmx_set_guest_uret_msr(vmx, msr, enabled ? 0 : TSX_CTRL_RTM_DISABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7300) set_cr4_guest_host_mask(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7302) /* Refresh #PF interception to account for MAXPHYADDR changes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7303) update_exception_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7306) static __init void vmx_set_cpu_caps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7308) kvm_set_cpu_caps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7310) /* CPUID 0x1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7311) if (nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7312) kvm_cpu_cap_set(X86_FEATURE_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7314) /* CPUID 0x7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7315) if (kvm_mpx_supported())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7316) kvm_cpu_cap_check_and_set(X86_FEATURE_MPX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7317) if (cpu_has_vmx_invpcid())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7318) kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7319) if (vmx_pt_mode_is_host_guest())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7320) kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7322) if (vmx_umip_emulated())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7323) kvm_cpu_cap_set(X86_FEATURE_UMIP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7325) /* CPUID 0xD.1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7326) supported_xss = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7327) if (!cpu_has_vmx_xsaves())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7328) kvm_cpu_cap_clear(X86_FEATURE_XSAVES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7330) /* CPUID 0x80000001 and 0x7 (RDPID) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7331) if (!cpu_has_vmx_rdtscp()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7332) kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7333) kvm_cpu_cap_clear(X86_FEATURE_RDPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7336) if (cpu_has_vmx_waitpkg())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7337) kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7340) static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7342) to_vmx(vcpu)->req_immediate_exit = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7345) static int vmx_check_intercept_io(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7346) struct x86_instruction_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7348) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7349) unsigned short port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7350) bool intercept;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7351) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7353) if (info->intercept == x86_intercept_in ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7354) info->intercept == x86_intercept_ins) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7355) port = info->src_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7356) size = info->dst_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7357) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7358) port = info->dst_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7359) size = info->src_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7363) * If the 'use IO bitmaps' VM-execution control is 0, IO instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7364) * VM-exits depend on the 'unconditional IO exiting' VM-execution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7365) * control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7366) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7367) * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7369) if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7370) intercept = nested_cpu_has(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7371) CPU_BASED_UNCOND_IO_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7372) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7373) intercept = nested_vmx_check_io_bitmaps(vcpu, port, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7375) /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7376) return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7379) static int vmx_check_intercept(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7380) struct x86_instruction_info *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7381) enum x86_intercept_stage stage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7382) struct x86_exception *exception)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7384) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7386) switch (info->intercept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7387) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7388) * RDPID causes #UD if disabled through secondary execution controls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7389) * Because it is marked as EmulateOnUD, we need to intercept it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7390) * Note, RDPID is hidden behind ENABLE_RDTSCP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7392) case x86_intercept_rdpid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7393) if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_RDTSCP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7394) exception->vector = UD_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7395) exception->error_code_valid = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7396) return X86EMUL_PROPAGATE_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7398) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7400) case x86_intercept_in:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7401) case x86_intercept_ins:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7402) case x86_intercept_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7403) case x86_intercept_outs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7404) return vmx_check_intercept_io(vcpu, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7406) case x86_intercept_lgdt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7407) case x86_intercept_lidt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7408) case x86_intercept_lldt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7409) case x86_intercept_ltr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7410) case x86_intercept_sgdt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7411) case x86_intercept_sidt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7412) case x86_intercept_sldt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7413) case x86_intercept_str:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7414) if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7415) return X86EMUL_CONTINUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7417) /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7418) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7420) /* TODO: check more intercepts... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7421) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7422) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7425) return X86EMUL_UNHANDLEABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7428) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7429) /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7430) static inline int u64_shl_div_u64(u64 a, unsigned int shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7431) u64 divisor, u64 *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7433) u64 low = a << shift, high = a >> (64 - shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7435) /* To avoid the overflow on divq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7436) if (high >= divisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7437) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7439) /* Low hold the result, high hold rem which is discarded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7440) asm("divq %2\n\t" : "=a" (low), "=d" (high) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7441) "rm" (divisor), "0" (low), "1" (high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7442) *result = low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7444) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7447) static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7448) bool *expired)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7450) struct vcpu_vmx *vmx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7451) u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7452) struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7454) vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7455) tscl = rdtsc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7456) guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7457) delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7458) lapic_timer_advance_cycles = nsec_to_cycles(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7459) ktimer->timer_advance_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7461) if (delta_tsc > lapic_timer_advance_cycles)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7462) delta_tsc -= lapic_timer_advance_cycles;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7463) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7464) delta_tsc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7466) /* Convert to host delta tsc if tsc scaling is enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7467) if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7468) delta_tsc && u64_shl_div_u64(delta_tsc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7469) kvm_tsc_scaling_ratio_frac_bits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7470) vcpu->arch.tsc_scaling_ratio, &delta_tsc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7471) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7474) * If the delta tsc can't fit in the 32 bit after the multi shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7475) * we can't use the preemption timer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7476) * It's possible that it fits on later vmentries, but checking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7477) * on every vmentry is costly so we just use an hrtimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7478) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7479) if (delta_tsc >> (cpu_preemption_timer_multi + 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7480) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7482) vmx->hv_deadline_tsc = tscl + delta_tsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7483) *expired = !delta_tsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7484) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7487) static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7489) to_vmx(vcpu)->hv_deadline_tsc = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7491) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7493) static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7495) if (!kvm_pause_in_guest(vcpu->kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7496) shrink_ple_window(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7499) static void vmx_slot_enable_log_dirty(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7500) struct kvm_memory_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7502) if (!kvm_dirty_log_manual_protect_and_init_set(kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7503) kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7504) kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7507) static void vmx_slot_disable_log_dirty(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7508) struct kvm_memory_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7510) kvm_mmu_slot_set_dirty(kvm, slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7513) static void vmx_flush_log_dirty(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7515) kvm_flush_pml_buffers(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7518) static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7519) struct kvm_memory_slot *memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7520) gfn_t offset, unsigned long mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7522) kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7525) static int vmx_pre_block(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7527) if (pi_pre_block(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7528) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7530) if (kvm_lapic_hv_timer_in_use(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7531) kvm_lapic_switch_to_sw_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7533) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7536) static void vmx_post_block(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7538) if (kvm_x86_ops.set_hv_timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7539) kvm_lapic_switch_to_hv_timer(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7541) pi_post_block(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7544) static void vmx_setup_mce(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7546) if (vcpu->arch.mcg_cap & MCG_LMCE_P)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7547) to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7548) FEAT_CTL_LMCE_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7549) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7550) to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7551) ~FEAT_CTL_LMCE_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7554) static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7556) /* we need a nested vmexit to enter SMM, postpone if run is pending */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7557) if (to_vmx(vcpu)->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7558) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7559) return !is_smm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7562) static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7564) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7566) vmx->nested.smm.guest_mode = is_guest_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7567) if (vmx->nested.smm.guest_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7568) nested_vmx_vmexit(vcpu, -1, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7570) vmx->nested.smm.vmxon = vmx->nested.vmxon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7571) vmx->nested.vmxon = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7572) vmx_clear_hlt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7573) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7576) static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7578) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7579) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7581) if (vmx->nested.smm.vmxon) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7582) vmx->nested.vmxon = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7583) vmx->nested.smm.vmxon = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7586) if (vmx->nested.smm.guest_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7587) ret = nested_vmx_enter_non_root_mode(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7588) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7589) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7591) vmx->nested.smm.guest_mode = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7593) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7596) static void enable_smi_window(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7598) /* RSM will cause a vmexit anyway. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7601) static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7603) return to_vmx(vcpu)->nested.vmxon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7606) static void vmx_migrate_timers(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7608) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7609) struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7611) if (hrtimer_try_to_cancel(timer) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7612) hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7616) static void hardware_unsetup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7618) kvm_set_posted_intr_wakeup_handler(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7620) if (nested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7621) nested_vmx_hardware_unsetup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7623) free_kvm_area();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7626) static bool vmx_check_apicv_inhibit_reasons(ulong bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7628) ulong supported = BIT(APICV_INHIBIT_REASON_DISABLE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7629) BIT(APICV_INHIBIT_REASON_HYPERV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7631) return supported & BIT(bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7634) static struct kvm_x86_ops vmx_x86_ops __initdata = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7635) .hardware_unsetup = hardware_unsetup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7637) .hardware_enable = hardware_enable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7638) .hardware_disable = hardware_disable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7639) .cpu_has_accelerated_tpr = report_flexpriority,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7640) .has_emulated_msr = vmx_has_emulated_msr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7642) .vm_size = sizeof(struct kvm_vmx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7643) .vm_init = vmx_vm_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7645) .vcpu_create = vmx_create_vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7646) .vcpu_free = vmx_free_vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7647) .vcpu_reset = vmx_vcpu_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7649) .prepare_guest_switch = vmx_prepare_switch_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7650) .vcpu_load = vmx_vcpu_load,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7651) .vcpu_put = vmx_vcpu_put,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7653) .update_exception_bitmap = update_exception_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7654) .get_msr_feature = vmx_get_msr_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7655) .get_msr = vmx_get_msr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7656) .set_msr = vmx_set_msr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7657) .get_segment_base = vmx_get_segment_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7658) .get_segment = vmx_get_segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7659) .set_segment = vmx_set_segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7660) .get_cpl = vmx_get_cpl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7661) .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7662) .set_cr0 = vmx_set_cr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7663) .set_cr4 = vmx_set_cr4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7664) .set_efer = vmx_set_efer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7665) .get_idt = vmx_get_idt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7666) .set_idt = vmx_set_idt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7667) .get_gdt = vmx_get_gdt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7668) .set_gdt = vmx_set_gdt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7669) .set_dr7 = vmx_set_dr7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7670) .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7671) .cache_reg = vmx_cache_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7672) .get_rflags = vmx_get_rflags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7673) .set_rflags = vmx_set_rflags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7675) .tlb_flush_all = vmx_flush_tlb_all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7676) .tlb_flush_current = vmx_flush_tlb_current,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7677) .tlb_flush_gva = vmx_flush_tlb_gva,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7678) .tlb_flush_guest = vmx_flush_tlb_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7680) .run = vmx_vcpu_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7681) .handle_exit = vmx_handle_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7682) .skip_emulated_instruction = vmx_skip_emulated_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7683) .update_emulated_instruction = vmx_update_emulated_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7684) .set_interrupt_shadow = vmx_set_interrupt_shadow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7685) .get_interrupt_shadow = vmx_get_interrupt_shadow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7686) .patch_hypercall = vmx_patch_hypercall,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7687) .set_irq = vmx_inject_irq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7688) .set_nmi = vmx_inject_nmi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7689) .queue_exception = vmx_queue_exception,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7690) .cancel_injection = vmx_cancel_injection,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7691) .interrupt_allowed = vmx_interrupt_allowed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7692) .nmi_allowed = vmx_nmi_allowed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7693) .get_nmi_mask = vmx_get_nmi_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7694) .set_nmi_mask = vmx_set_nmi_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7695) .enable_nmi_window = enable_nmi_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7696) .enable_irq_window = enable_irq_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7697) .update_cr8_intercept = update_cr8_intercept,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7698) .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7699) .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7700) .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7701) .load_eoi_exitmap = vmx_load_eoi_exitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7702) .apicv_post_state_restore = vmx_apicv_post_state_restore,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7703) .check_apicv_inhibit_reasons = vmx_check_apicv_inhibit_reasons,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7704) .hwapic_irr_update = vmx_hwapic_irr_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7705) .hwapic_isr_update = vmx_hwapic_isr_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7706) .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7707) .sync_pir_to_irr = vmx_sync_pir_to_irr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7708) .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7709) .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7711) .set_tss_addr = vmx_set_tss_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7712) .set_identity_map_addr = vmx_set_identity_map_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7713) .get_mt_mask = vmx_get_mt_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7715) .get_exit_info = vmx_get_exit_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7717) .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7719) .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7721) .write_l1_tsc_offset = vmx_write_l1_tsc_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7723) .load_mmu_pgd = vmx_load_mmu_pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7725) .check_intercept = vmx_check_intercept,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7726) .handle_exit_irqoff = vmx_handle_exit_irqoff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7728) .request_immediate_exit = vmx_request_immediate_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7730) .sched_in = vmx_sched_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7732) .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7733) .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7734) .flush_log_dirty = vmx_flush_log_dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7735) .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7737) .pre_block = vmx_pre_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7738) .post_block = vmx_post_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7740) .pmu_ops = &intel_pmu_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7741) .nested_ops = &vmx_nested_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7743) .update_pi_irte = pi_update_irte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7745) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7746) .set_hv_timer = vmx_set_hv_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7747) .cancel_hv_timer = vmx_cancel_hv_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7748) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7750) .setup_mce = vmx_setup_mce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7752) .smi_allowed = vmx_smi_allowed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7753) .pre_enter_smm = vmx_pre_enter_smm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7754) .pre_leave_smm = vmx_pre_leave_smm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7755) .enable_smi_window = enable_smi_window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7757) .can_emulate_instruction = vmx_can_emulate_instruction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7758) .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7759) .migrate_timers = vmx_migrate_timers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7761) .msr_filter_changed = vmx_msr_filter_changed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7762) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7764) static __init int hardware_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7766) unsigned long host_bndcfgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7767) struct desc_ptr dt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7768) int r, i, ept_lpage_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7770) store_idt(&dt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7771) host_idt_base = dt.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7773) for (i = 0; i < ARRAY_SIZE(vmx_uret_msrs_list); ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7774) kvm_define_user_return_msr(i, vmx_uret_msrs_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7776) if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7777) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7779) if (boot_cpu_has(X86_FEATURE_NX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7780) kvm_enable_efer_bits(EFER_NX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7782) if (boot_cpu_has(X86_FEATURE_MPX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7783) rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7784) WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7787) if (!cpu_has_vmx_mpx())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7788) supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7789) XFEATURE_MASK_BNDCSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7791) if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7792) !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7793) enable_vpid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7795) if (!cpu_has_vmx_ept() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7796) !cpu_has_vmx_ept_4levels() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7797) !cpu_has_vmx_ept_mt_wb() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7798) !cpu_has_vmx_invept_global())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7799) enable_ept = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7801) if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7802) enable_ept_ad_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7804) if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7805) enable_unrestricted_guest = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7807) if (!cpu_has_vmx_flexpriority())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7808) flexpriority_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7810) if (!cpu_has_virtual_nmis())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7811) enable_vnmi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7814) * set_apic_access_page_addr() is used to reload apic access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7815) * page upon invalidation. No need to do anything if not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7816) * using the APIC_ACCESS_ADDR VMCS field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7817) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7818) if (!flexpriority_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7819) vmx_x86_ops.set_apic_access_page_addr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7821) if (!cpu_has_vmx_tpr_shadow())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7822) vmx_x86_ops.update_cr8_intercept = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7824) #if IS_ENABLED(CONFIG_HYPERV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7825) if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7826) && enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7827) vmx_x86_ops.tlb_remote_flush = hv_remote_flush_tlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7828) vmx_x86_ops.tlb_remote_flush_with_range =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7829) hv_remote_flush_tlb_with_range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7831) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7833) if (!cpu_has_vmx_ple()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7834) ple_gap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7835) ple_window = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7836) ple_window_grow = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7837) ple_window_max = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7838) ple_window_shrink = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7841) if (!cpu_has_vmx_apicv()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7842) enable_apicv = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7843) vmx_x86_ops.sync_pir_to_irr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7846) if (cpu_has_vmx_tsc_scaling()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7847) kvm_has_tsc_control = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7848) kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7849) kvm_tsc_scaling_ratio_frac_bits = 48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7852) set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7854) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7855) vmx_enable_tdp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7857) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7858) ept_lpage_level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7859) else if (cpu_has_vmx_ept_1g_page())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7860) ept_lpage_level = PG_LEVEL_1G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7861) else if (cpu_has_vmx_ept_2m_page())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7862) ept_lpage_level = PG_LEVEL_2M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7863) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7864) ept_lpage_level = PG_LEVEL_4K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7865) kvm_configure_mmu(enable_ept, vmx_get_max_tdp_level(), ept_lpage_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7867) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7868) * Only enable PML when hardware supports PML feature, and both EPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7869) * and EPT A/D bit features are enabled -- PML depends on them to work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7871) if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7872) enable_pml = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7874) if (!enable_pml) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7875) vmx_x86_ops.slot_enable_log_dirty = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7876) vmx_x86_ops.slot_disable_log_dirty = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7877) vmx_x86_ops.flush_log_dirty = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7878) vmx_x86_ops.enable_log_dirty_pt_masked = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7881) if (!cpu_has_vmx_preemption_timer())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7882) enable_preemption_timer = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7884) if (enable_preemption_timer) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7885) u64 use_timer_freq = 5000ULL * 1000 * 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7886) u64 vmx_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7888) rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7889) cpu_preemption_timer_multi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7890) vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7892) if (tsc_khz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7893) use_timer_freq = (u64)tsc_khz * 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7894) use_timer_freq >>= cpu_preemption_timer_multi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7896) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7897) * KVM "disables" the preemption timer by setting it to its max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7898) * value. Don't use the timer if it might cause spurious exits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7899) * at a rate faster than 0.1 Hz (of uninterrupted guest time).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7900) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7901) if (use_timer_freq > 0xffffffffu / 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7902) enable_preemption_timer = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7905) if (!enable_preemption_timer) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7906) vmx_x86_ops.set_hv_timer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7907) vmx_x86_ops.cancel_hv_timer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7908) vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7911) kvm_mce_cap_supported |= MCG_LMCE_P;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7913) if (pt_mode != PT_MODE_SYSTEM && pt_mode != PT_MODE_HOST_GUEST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7914) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7915) if (!enable_ept || !cpu_has_vmx_intel_pt())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7916) pt_mode = PT_MODE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7918) if (nested) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7919) nested_vmx_setup_ctls_msrs(&vmcs_config.nested,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7920) vmx_capability.ept);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7922) r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7923) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7924) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7927) vmx_set_cpu_caps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7929) r = alloc_kvm_area();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7930) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7931) nested_vmx_hardware_unsetup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7933) kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7935) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7938) static struct kvm_x86_init_ops vmx_init_ops __initdata = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7939) .cpu_has_kvm_support = cpu_has_kvm_support,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7940) .disabled_by_bios = vmx_disabled_by_bios,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7941) .check_processor_compatibility = vmx_check_processor_compat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7942) .hardware_setup = hardware_setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7943) .intel_pt_intr_in_guest = vmx_pt_mode_is_host_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7945) .runtime_ops = &vmx_x86_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7946) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7948) static void vmx_cleanup_l1d_flush(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7950) if (vmx_l1d_flush_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7951) free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7952) vmx_l1d_flush_pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7954) /* Restore state so sysfs ignores VMX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7955) l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7958) static void vmx_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7960) #ifdef CONFIG_KEXEC_CORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7961) RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7962) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7963) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7965) kvm_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7967) #if IS_ENABLED(CONFIG_HYPERV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7968) if (static_branch_unlikely(&enable_evmcs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7969) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7970) struct hv_vp_assist_page *vp_ap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7971) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7972) * Reset everything to support using non-enlightened VMCS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7973) * access later (e.g. when we reload the module with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7974) * enlightened_vmcs=0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7975) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7976) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7977) vp_ap = hv_get_vp_assist_page(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7979) if (!vp_ap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7980) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7982) vp_ap->nested_control.features.directhypercall = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7983) vp_ap->current_nested_vmcs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7984) vp_ap->enlighten_vmentry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7987) static_branch_disable(&enable_evmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7989) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7990) vmx_cleanup_l1d_flush();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7992) module_exit(vmx_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7994) static int __init vmx_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7995) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7996) int r, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7998) #if IS_ENABLED(CONFIG_HYPERV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8000) * Enlightened VMCS usage should be recommended and the host needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8001) * to support eVMCS v1 or above. We can also disable eVMCS support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8002) * with module parameter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8003) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8004) if (enlightened_vmcs &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8005) ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8006) (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8007) KVM_EVMCS_VERSION) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8008) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8010) /* Check that we have assist pages on all online CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8011) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8012) if (!hv_get_vp_assist_page(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8013) enlightened_vmcs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8014) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8018) if (enlightened_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8019) pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8020) static_branch_enable(&enable_evmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8023) if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8024) vmx_x86_ops.enable_direct_tlbflush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8025) = hv_enable_direct_tlbflush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8027) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8028) enlightened_vmcs = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8030) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8032) r = kvm_init(&vmx_init_ops, sizeof(struct vcpu_vmx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8033) __alignof__(struct vcpu_vmx), THIS_MODULE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8034) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8035) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8037) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8038) * Must be called after kvm_init() so enable_ept is properly set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8039) * up. Hand the parameter mitigation value in which was stored in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8040) * the pre module init parser. If no parameter was given, it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8041) * contain 'auto' which will be turned into the default 'cond'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8042) * mitigation mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8044) r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8045) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8046) vmx_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8047) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8050) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8051) INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8053) pi_init_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8056) #ifdef CONFIG_KEXEC_CORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8057) rcu_assign_pointer(crash_vmclear_loaded_vmcss,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8058) crash_vmclear_local_loaded_vmcss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8059) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8060) vmx_check_vmcs12_offsets();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8063) * Shadow paging doesn't have a (further) performance penalty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8064) * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8065) * by default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8066) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8067) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8068) allow_smaller_maxphyaddr = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8070) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8072) module_init(vmx_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags