^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/objtool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/debugreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "cpuid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "hyperv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "mmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "nested.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "pmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) static bool __read_mostly enable_shadow_vmcs = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static bool __read_mostly nested_early_check = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) module_param(nested_early_check, bool, S_IRUGO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define CC(consistency_check) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) bool failed = (consistency_check); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) if (failed) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) failed; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * Hyper-V requires all of these, so mark them as supported even though
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * they are just treated the same as all-context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define VMX_VPID_EXTENT_SUPPORTED_MASK \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) VMX_VMREAD_BITMAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) VMX_VMWRITE_BITMAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) VMX_BITMAP_NR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct shadow_vmcs_field {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u16 encoding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) u16 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) static struct shadow_vmcs_field shadow_read_only_fields[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "vmcs_shadow_fields.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static int max_shadow_read_only_fields =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) ARRAY_SIZE(shadow_read_only_fields);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static struct shadow_vmcs_field shadow_read_write_fields[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include "vmcs_shadow_fields.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static int max_shadow_read_write_fields =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ARRAY_SIZE(shadow_read_write_fields);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static void init_vmcs_shadow_fields(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) for (i = j = 0; i < max_shadow_read_only_fields; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct shadow_vmcs_field entry = shadow_read_only_fields[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u16 field = entry.encoding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) (i + 1 == max_shadow_read_only_fields ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) shadow_read_only_fields[i + 1].encoding != field + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) pr_err("Missing field from shadow_read_only_field %x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) field + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) clear_bit(field, vmx_vmread_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (field & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) entry.offset += sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) shadow_read_only_fields[j++] = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) max_shadow_read_only_fields = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) for (i = j = 0; i < max_shadow_read_write_fields; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct shadow_vmcs_field entry = shadow_read_write_fields[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) u16 field = entry.encoding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) (i + 1 == max_shadow_read_write_fields ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) shadow_read_write_fields[i + 1].encoding != field + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) pr_err("Missing field from shadow_read_write_field %x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) field + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) WARN_ONCE(field >= GUEST_ES_AR_BYTES &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) field <= GUEST_TR_AR_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) "Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * PML and the preemption timer can be emulated, but the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * processor cannot vmwrite to fields that don't exist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * on bare metal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) switch (field) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) case GUEST_PML_INDEX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (!cpu_has_vmx_pml())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) case VMX_PREEMPTION_TIMER_VALUE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (!cpu_has_vmx_preemption_timer())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) case GUEST_INTR_STATUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (!cpu_has_vmx_apicv())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) clear_bit(field, vmx_vmwrite_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) clear_bit(field, vmx_vmread_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (field & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) entry.offset += sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) shadow_read_write_fields[j++] = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) max_shadow_read_write_fields = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * set the success or error code of an emulated VMX instruction (as specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static int nested_vmx_succeed(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) X86_EFLAGS_SF | X86_EFLAGS_OF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) | X86_EFLAGS_CF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static int nested_vmx_failValid(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) u32 vm_instruction_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) X86_EFLAGS_SF | X86_EFLAGS_OF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) | X86_EFLAGS_ZF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * We don't need to force a shadow sync because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * VM_INSTRUCTION_ERROR is not shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * failValid writes the error number to the current VMCS, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * can't be done if there isn't a current VMCS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return nested_vmx_failValid(vcpu, vm_instruction_error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /* TODO: not to reset guest simply here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return fixed_bits_valid(control, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline u64 vmx_control_msr(u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return low | ((u64)high << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) vmcs_write64(VMCS_LINK_POINTER, -1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) vmx->nested.need_vmcs12_to_shadow_sync = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) static inline void nested_release_evmcs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (!vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) vmx->nested.hv_evmcs_vmptr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) vmx->nested.hv_evmcs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct loaded_vmcs *prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) struct vmcs_host_state *dest, *src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (unlikely(!vmx->guest_state_loaded))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) src = &prev->host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) dest = &vmx->loaded_vmcs->host_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) dest->ldt_sel = src->ldt_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) dest->ds_sel = src->ds_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) dest->es_sel = src->es_sel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct loaded_vmcs *prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) prev = vmx->loaded_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) vmx->loaded_vmcs = vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) vmx_vcpu_load_vmcs(vcpu, cpu, prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) vmx_sync_vmcs_host_state(vmx, prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) vmx_register_cache_reset(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * Free whatever needs to be freed from vmx->nested when L1 goes down, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * just stops using VMX.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static void free_nested(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) vmx_switch_vmcs(vcpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) vmx->nested.vmxon = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) vmx->nested.smm.vmxon = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) free_vpid(vmx->nested.vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) vmx->nested.posted_intr_nv = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) vmx->nested.current_vmptr = -1ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) vmx_disable_shadow_vmcs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) vmcs_clear(vmx->vmcs01.shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) free_vmcs(vmx->vmcs01.shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) vmx->vmcs01.shadow_vmcs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) kfree(vmx->nested.cached_vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) vmx->nested.cached_vmcs12 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) kfree(vmx->nested.cached_shadow_vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) vmx->nested.cached_shadow_vmcs12 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /* Unpin physical memory we referred to in the vmcs02 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (vmx->nested.apic_access_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) kvm_release_page_clean(vmx->nested.apic_access_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) vmx->nested.apic_access_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) vmx->nested.pi_desc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) nested_release_evmcs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) free_loaded_vmcs(&vmx->nested.vmcs02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Ensure that the current vmcs of the logical processor is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * vmcs01 of the vcpu before calling free_nested().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) vcpu_load(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) vmx_leave_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) vcpu_put(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct x86_exception *fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) u32 vm_exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) unsigned long exit_qualification = vcpu->arch.exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (vmx->nested.pml_full) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) vm_exit_reason = EXIT_REASON_PML_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) vmx->nested.pml_full = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) exit_qualification &= INTR_INFO_UNBLOCK_NMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) } else if (fault->error_code & PFERR_RSVD_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) vm_exit_reason = EXIT_REASON_EPT_MISCONFIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) vm_exit_reason = EXIT_REASON_EPT_VIOLATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) vmcs12->guest_physical_address = fault->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) WARN_ON(mmu_is_nested(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) vcpu->arch.mmu = &vcpu->arch.guest_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) kvm_init_shadow_ept_mmu(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) to_vmx(vcpu)->nested.msrs.ept_caps &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) VMX_EPT_EXECUTE_ONLY_BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) nested_ept_ad_enabled(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) nested_ept_get_eptp(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) vcpu->arch.mmu->get_pdptr = kvm_pdptr_read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) vcpu->arch.mmu = &vcpu->arch.root_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) u16 error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) bool inequality, bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) inequality =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) (error_code & vmcs12->page_fault_error_code_mask) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) vmcs12->page_fault_error_code_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return inequality ^ bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * KVM wants to inject page-faults which it got to the guest. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * checks whether in a nested guest, we need to inject them to L1 or L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) unsigned int nr = vcpu->arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) bool has_payload = vcpu->arch.exception.has_payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) unsigned long payload = vcpu->arch.exception.payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (nr == PF_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (vcpu->arch.exception.nested_apf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) *exit_qual = vcpu->arch.apf.nested_apf_token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (nested_vmx_is_page_fault_vmexit(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) vcpu->arch.exception.error_code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) *exit_qual = has_payload ? payload : vcpu->arch.cr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) } else if (vmcs12->exception_bitmap & (1u << nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (nr == DB_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (!has_payload) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) payload = vcpu->arch.dr6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) payload &= ~(DR6_FIXED_1 | DR6_BT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) payload ^= DR6_RTM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) *exit_qual = payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) *exit_qual = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct x86_exception *fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) WARN_ON(!is_guest_mode(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) !to_vmx(vcpu)->nested.nested_run_pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) vmcs12->vm_exit_intr_error_code = fault->error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) fault->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) kvm_inject_page_fault(vcpu, fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * Check if MSR is intercepted for L01 MSR bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) unsigned long *msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (!cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (msr <= 0x1fff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return !!test_bit(msr, msr_bitmap + 0x800 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) msr &= 0x1fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return !!test_bit(msr, msr_bitmap + 0xc00 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * If a msr is allowed by L0, we should check whether it is allowed by L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * The corresponding bit will be cleared unless both of L0 and L1 allow it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) unsigned long *msr_bitmap_nested,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) u32 msr, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) int f = sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * have the write-low and read-high bitmap offsets the wrong way round.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (msr <= 0x1fff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (type & MSR_TYPE_R &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* read-low */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) if (type & MSR_TYPE_W &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /* write-low */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) msr &= 0x1fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (type & MSR_TYPE_R &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /* read-high */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (type & MSR_TYPE_W &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /* write-high */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) int msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) unsigned word = msr / BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) msr_bitmap[word] = ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * Merge L0's and L1's MSR bitmap, return false to indicate that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * we do not use the hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) int msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) unsigned long *msr_bitmap_l1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /* Nothing to do if the MSR bitmap is not in use. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) if (!cpu_has_vmx_msr_bitmap() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) msr_bitmap_l1 = (unsigned long *)map->hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * To keep the control flow simple, pay eight 8-byte writes (sixteen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * 4-byte writes on 32-bit systems) up front to enable intercepts for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * the x2APIC MSR range and selectively disable them below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) enable_x2apic_msr_intercepts(msr_bitmap_l0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (nested_cpu_has_apic_reg_virt(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * L0 need not intercept reads for MSRs between 0x800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * and 0x8ff, it just lets the processor take the value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * from the virtual-APIC page; take those 256 bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * directly from the L1 bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) unsigned word = msr / BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) msr_bitmap_l0[word] = msr_bitmap_l1[word];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) nested_vmx_disable_intercept_for_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) X2APIC_MSR(APIC_TASKPRI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) MSR_TYPE_R | MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (nested_cpu_has_vid(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) nested_vmx_disable_intercept_for_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) X2APIC_MSR(APIC_EOI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) nested_vmx_disable_intercept_for_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) X2APIC_MSR(APIC_SELF_IPI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /* KVM unconditionally exposes the FS/GS base MSRs to L1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) MSR_FS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) MSR_GS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Checking the L0->L1 bitmap is trying to verify two things:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * ensures that we do not accidentally generate an L02 MSR bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * from the L12 MSR bitmap that is too permissive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * 2. That L1 or L2s have actually used the MSR. This avoids
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * unnecessarily merging of the bitmap if the MSR is unused. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * works properly because we only update the L01 MSR bitmap lazily.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * So even if L0 should pass L1 these MSRs, the L01 bitmap is only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * updated to reflect this when L1 (or its L2s) actually write to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * the MSR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) nested_vmx_disable_intercept_for_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) MSR_IA32_SPEC_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) MSR_TYPE_R | MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) nested_vmx_disable_intercept_for_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) msr_bitmap_l1, msr_bitmap_l0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) MSR_IA32_PRED_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) MSR_TYPE_W);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct kvm_host_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) struct vmcs12 *shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) vmcs12->vmcs_link_pointer == -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) shadow = get_shadow_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) memcpy(shadow, map.hva, VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) kvm_vcpu_unmap(vcpu, &map, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) vmcs12->vmcs_link_pointer == -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) get_shadow_vmcs12(vcpu), VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * In nested virtualization, check if L1 has set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * VM_EXIT_ACK_INTR_ON_EXIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) return get_vmcs12(vcpu)->vm_exit_controls &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) VM_EXIT_ACK_INTR_ON_EXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) !nested_cpu_has_apic_reg_virt(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) !nested_cpu_has_vid(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) !nested_cpu_has_posted_intr(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * If virtualize x2apic mode is enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * virtualize apic access must be disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * If virtual interrupt delivery is enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * we must exit on external interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * bits 15:8 should be zero in posted_intr_nv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * the descriptor address has been already checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * in nested_get_vmcs12_pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * bits 5:0 of posted_intr_desc_addr should be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (nested_cpu_has_posted_intr(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) (CC(!nested_cpu_has_vid(vmcs12)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) CC(!nested_exit_intr_ack_set(vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) CC((vmcs12->posted_intr_nv & 0xff00)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) CC((vmcs12->posted_intr_desc_addr & 0x3f)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /* tpr shadow is needed by all apicv features. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) u32 count, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) int maxphyaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) if (count == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) maxphyaddr = cpuid_maxphyaddr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (CC(nested_vmx_check_msr_switch(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) vmcs12->vm_exit_msr_load_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) vmcs12->vm_exit_msr_load_addr)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) CC(nested_vmx_check_msr_switch(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) vmcs12->vm_exit_msr_store_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) vmcs12->vm_exit_msr_store_addr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (CC(nested_vmx_check_msr_switch(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) vmcs12->vm_entry_msr_load_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) vmcs12->vm_entry_msr_load_addr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (!nested_cpu_has_pml(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (CC(!nested_cpu_has_ept(vmcs12)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) CC(!page_address_valid(vcpu, vmcs12->pml_address)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) !nested_cpu_has_ept(vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) !nested_cpu_has_ept(vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (!nested_cpu_has_shadow_vmcs(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct vmx_msr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /* x2APIC MSR accesses are not allowed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) CC(e->index == MSR_IA32_UCODE_REV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) if (CC(e->reserved != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) struct vmx_msr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (CC(e->index == MSR_FS_BASE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) CC(e->index == MSR_GS_BASE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) nested_vmx_msr_check_common(vcpu, e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) struct vmx_msr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) nested_vmx_msr_check_common(vcpu, e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) vmx->nested.msrs.misc_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * Load guest's/host's msr at nested entry/exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * return 0 for success, entry index for failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * One of the failure modes for MSR load/store is when a list exceeds the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * virtual hardware's capacity. To maintain compatibility with hardware inasmuch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * as possible, process all valid entries before failing rather than precheck
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * for a capacity violation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) struct vmx_msr_entry e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (unlikely(i >= max_msr_list_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) &e, sizeof(e))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) "%s cannot read MSR entry (%u, 0x%08llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) __func__, i, gpa + i * sizeof(e));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (nested_vmx_load_msr_check(vcpu, &e)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) "%s check failed (%u, 0x%x, 0x%x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) __func__, i, e.index, e.reserved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) if (kvm_set_msr(vcpu, e.index, e.value)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) __func__, i, e.index, e.value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) u32 msr_index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) u64 *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * If the L0 hypervisor stored a more accurate value for the TSC that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * does not include the time taken for emulation of the L2->L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * VM-exit in L0, use the more accurate value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (msr_index == MSR_IA32_TSC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) MSR_IA32_TSC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (i >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) u64 val = vmx->msr_autostore.guest.val[i].value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) *data = kvm_read_l1_tsc(vcpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) if (kvm_get_msr(vcpu, msr_index, data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) msr_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct vmx_msr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (kvm_vcpu_read_guest(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) gpa + i * sizeof(*e),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) e, 2 * sizeof(u32))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) "%s cannot read MSR entry (%u, 0x%08llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) __func__, i, gpa + i * sizeof(*e));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (nested_vmx_store_msr_check(vcpu, e)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) "%s check failed (%u, 0x%x, 0x%x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) __func__, i, e->index, e->reserved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) u64 data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct vmx_msr_entry e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (unlikely(i >= max_msr_list_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (kvm_vcpu_write_guest(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) gpa + i * sizeof(e) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) offsetof(struct vmx_msr_entry, value),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) &data, sizeof(data))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) __func__, i, e.index, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) u32 count = vmcs12->vm_exit_msr_store_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) u64 gpa = vmcs12->vm_exit_msr_store_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) struct vmx_msr_entry e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (!read_and_check_msr_entry(vcpu, gpa, i, &e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (e.index == msr_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) u32 msr_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) struct vmx_msrs *autostore = &vmx->msr_autostore.guest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) bool in_vmcs12_store_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) int msr_autostore_slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) bool in_autostore_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) int last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) in_autostore_list = msr_autostore_slot >= 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (in_vmcs12_store_list && !in_autostore_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (autostore->nr == MAX_NR_LOADSTORE_MSRS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * Emulated VMEntry does not fail here. Instead a less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) * accurate value will be returned by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) * nested_vmx_get_vmexit_msr_value() using kvm_get_msr()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * instead of reading the value from the vmcs02 VMExit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * MSR-store area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) pr_warn_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) "Not enough msr entries in msr_autostore. Can't add msr %x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) msr_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) last = autostore->nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) autostore->val[last].index = msr_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) } else if (!in_vmcs12_store_list && in_autostore_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) last = --autostore->nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) autostore->val[msr_autostore_slot] = autostore->val[last];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) unsigned long invalid_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return (val & invalid_mask) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) * enable VPID for L2 (implying it expects a TLB flush on VMX transitions).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * Here's why.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * If EPT is enabled by L0 a sync is never needed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * - if it is disabled by L1, then L0 is not shadowing L1 or L2 PTEs, there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * cannot be unsync'd SPTEs for either L1 or L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * - if it is also enabled by L1, then L0 doesn't need to sync on VM-Enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * VM-Enter as VM-Enter isn't required to invalidate guest-physical mappings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * (irrespective of VPID), i.e. L1 can't rely on the (virtual) CPU to flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * stale guest-physical mappings for L2 from the TLB. And as above, L0 isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * shadowing L1 PTEs so there are no unsync'd SPTEs to sync on VM-Exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * If EPT is disabled by L0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * - if VPID is enabled by L1 (for L2), the situation is similar to when L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * enables EPT: L0 doesn't need to sync as VM-Enter and VM-Exit aren't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * required to invalidate linear mappings (EPT is disabled so there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * no combined or guest-physical mappings), i.e. L1 can't rely on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * (virtual) CPU to flush stale linear mappings for either L2 or itself (L1).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * - however if VPID is disabled by L1, then a sync is needed as L1 expects all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * linear mappings (EPT is disabled so there are no combined or guest-physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * mappings) to be invalidated on both VM-Enter and VM-Exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * Note, this logic is subtly different than nested_has_guest_tlb_tag(), which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * additionally checks that L2 has been assigned a VPID (when EPT is disabled).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * Whether or not L2 has been assigned a VPID by L0 is irrelevant with respect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * to L1's expectations, e.g. L0 needs to invalidate hardware TLB entries if L2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * doesn't have a unique VPID to prevent reusing L1's entries (assuming L1 has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) * been assigned a VPID), but L0 doesn't need to do a MMU sync because L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) * doesn't expect stale (virtual) TLB entries to be flushed, i.e. L1 doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * know that L0 will flush the TLB and so L1 will do INVVPID as needed to flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) * stale TLB entries, at which point L0 will sync L2's MMU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return !enable_ept && !nested_cpu_has_vpid(get_vmcs12(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * emulating VM-Entry into a guest with EPT enabled. On failure, the expected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) * @entry_failure_code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) enum vm_entry_failure_code *entry_failure_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) if (CC(!nested_cr3_valid(vcpu, cr3))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) *entry_failure_code = ENTRY_FAIL_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) * If PAE paging and EPT are both on, CR3 is not used by the CPU and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) * must not be dereferenced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (!nested_ept && is_pae_paging(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) *entry_failure_code = ENTRY_FAIL_PDPTE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * Unconditionally skip the TLB flush on fast CR3 switch, all TLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * flushes are handled by nested_vmx_transition_tlb_flush().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) if (!nested_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) kvm_mmu_new_pgd(vcpu, cr3, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * A TLB flush on VM-Enter/VM-Exit flushes all linear mappings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * across all PCIDs, i.e. all PGDs need to be synchronized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) * See nested_vmx_transition_mmu_sync() for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (nested_vmx_transition_mmu_sync(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) vcpu->arch.cr3 = cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) kvm_init_mmu(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * Returns if KVM is able to config CPU to tag TLB entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * populated by L2 differently than TLB entries populated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * by L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * If L0 uses EPT, L1 and L2 run with different EPTP because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) * are tagged with different EPTP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) * with different VPID (L1 entries are tagged with vmx->vpid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) * while L2 entries are tagged with vmx->nested.vpid02).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) return enable_ept ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) bool is_vmenter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * If VPID is disabled, linear and combined mappings are flushed on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) * VM-Enter/VM-Exit, and guest-physical mappings are valid only for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * their associated EPTP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) if (!enable_vpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * for *all* contexts to be flushed on VM-Enter/VM-Exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * If VPID is enabled and used by vmc12, but L2 does not have a unique
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * a VPID for L2, flush the current context as the effective ASID is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * common to both L1 and L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * Defer the flush so that it runs after vmcs02.EPTP has been set by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * KVM_REQ_LOAD_MMU_PGD (if nested EPT is enabled) and to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) * redundant flushes further down the nested pipeline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * If a TLB flush isn't required due to any of the above, and vpid12 is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * changing then the new "virtual" VPID (vpid12) will reuse the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * "real" VPID (vpid02), and so needs to be sync'd. There is no direct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * all nested vCPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) if (!nested_cpu_has_vpid(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) } else if (!nested_has_guest_tlb_tag(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) } else if (is_vmenter &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) vmx->nested.last_vpid = vmcs12->virtual_processor_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) vpid_sync_context(nested_get_vpid02(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) superset &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) subset &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) return (superset | subset) == superset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) const u64 feature_and_reserved =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) /* feature (except bit 48; see below) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) /* reserved */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) u64 vmx_basic = vmx->nested.msrs.basic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * KVM does not emulate a version of VMX that constrains physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) * addresses of VMX structures (e.g. VMCS) to 32-bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (data & BIT_ULL(48))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) if (vmx_basic_vmcs_revision_id(vmx_basic) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) vmx_basic_vmcs_revision_id(data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) vmx->nested.msrs.basic = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) u64 supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) u32 *lowp, *highp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) switch (msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) lowp = &vmx->nested.msrs.pinbased_ctls_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) highp = &vmx->nested.msrs.pinbased_ctls_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) lowp = &vmx->nested.msrs.procbased_ctls_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) highp = &vmx->nested.msrs.procbased_ctls_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) case MSR_IA32_VMX_TRUE_EXIT_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) lowp = &vmx->nested.msrs.exit_ctls_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) highp = &vmx->nested.msrs.exit_ctls_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) lowp = &vmx->nested.msrs.entry_ctls_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) highp = &vmx->nested.msrs.entry_ctls_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) case MSR_IA32_VMX_PROCBASED_CTLS2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) lowp = &vmx->nested.msrs.secondary_ctls_low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) highp = &vmx->nested.msrs.secondary_ctls_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) supported = vmx_control_msr(*lowp, *highp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) /* Check must-be-1 bits are still 1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) /* Check must-be-0 bits are still 0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) *lowp = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) *highp = data >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) const u64 feature_and_reserved_bits =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) /* feature */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) /* reserved */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) GENMASK_ULL(13, 9) | BIT_ULL(31);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) u64 vmx_misc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) vmx->nested.msrs.misc_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) if ((vmx->nested.msrs.pinbased_ctls_high &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) PIN_BASED_VMX_PREEMPTION_TIMER) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) vmx_misc_preemption_timer_rate(data) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) vmx_misc_preemption_timer_rate(vmx_misc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) vmx->nested.msrs.misc_low = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) vmx->nested.msrs.misc_high = data >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) u64 vmx_ept_vpid_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) vmx->nested.msrs.vpid_caps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) /* Every bit is either reserved or a feature bit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) vmx->nested.msrs.ept_caps = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) vmx->nested.msrs.vpid_caps = data >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) u64 *msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) switch (msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) case MSR_IA32_VMX_CR0_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) msr = &vmx->nested.msrs.cr0_fixed0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) case MSR_IA32_VMX_CR4_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) msr = &vmx->nested.msrs.cr4_fixed0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) * 1 bits (which indicates bits which "must-be-1" during VMX operation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) * must be 1 in the restored value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) if (!is_bitwise_subset(data, *msr, -1ULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) *msr = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) * Called when userspace is restoring VMX MSRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) * Returns 0 on success, non-0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * Don't allow changes to the VMX capability MSRs while the vCPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * is in VMX operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) if (vmx->nested.vmxon)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) switch (msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) case MSR_IA32_VMX_BASIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) return vmx_restore_vmx_basic(vmx, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) case MSR_IA32_VMX_PINBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) case MSR_IA32_VMX_PROCBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) case MSR_IA32_VMX_EXIT_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) case MSR_IA32_VMX_ENTRY_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * The "non-true" VMX capability MSRs are generated from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * "true" MSRs, so we do not support restoring them directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) * If userspace wants to emulate VMX_BASIC[55]=0, userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) * should restore the "true" MSRs with the must-be-1 bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) * DEFAULT SETTINGS".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) case MSR_IA32_VMX_TRUE_EXIT_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) case MSR_IA32_VMX_PROCBASED_CTLS2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return vmx_restore_control_msr(vmx, msr_index, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) case MSR_IA32_VMX_MISC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return vmx_restore_vmx_misc(vmx, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) case MSR_IA32_VMX_CR0_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) case MSR_IA32_VMX_CR4_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return vmx_restore_fixed0_msr(vmx, msr_index, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) case MSR_IA32_VMX_CR0_FIXED1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) case MSR_IA32_VMX_CR4_FIXED1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) * These MSRs are generated based on the vCPU's CPUID, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) * do not support restoring them directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) case MSR_IA32_VMX_EPT_VPID_CAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) return vmx_restore_vmx_ept_vpid_cap(vmx, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) case MSR_IA32_VMX_VMCS_ENUM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) vmx->nested.msrs.vmcs_enum = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) case MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) if (data & ~vmx->nested.msrs.vmfunc_controls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) vmx->nested.msrs.vmfunc_controls = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * The rest of the VMX capability MSRs do not support restore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) /* Returns 0 on success, non-0 otherwise. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) switch (msr_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) case MSR_IA32_VMX_BASIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) *pdata = msrs->basic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) case MSR_IA32_VMX_PINBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) msrs->pinbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) msrs->pinbased_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) case MSR_IA32_VMX_PROCBASED_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) msrs->procbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) msrs->procbased_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) case MSR_IA32_VMX_TRUE_EXIT_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) case MSR_IA32_VMX_EXIT_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) msrs->exit_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) msrs->exit_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) case MSR_IA32_VMX_ENTRY_CTLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) msrs->entry_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) msrs->entry_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) case MSR_IA32_VMX_MISC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) msrs->misc_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) msrs->misc_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) case MSR_IA32_VMX_CR0_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) *pdata = msrs->cr0_fixed0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) case MSR_IA32_VMX_CR0_FIXED1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) *pdata = msrs->cr0_fixed1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) case MSR_IA32_VMX_CR4_FIXED0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) *pdata = msrs->cr4_fixed0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) case MSR_IA32_VMX_CR4_FIXED1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) *pdata = msrs->cr4_fixed1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) case MSR_IA32_VMX_VMCS_ENUM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) *pdata = msrs->vmcs_enum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) case MSR_IA32_VMX_PROCBASED_CTLS2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) *pdata = vmx_control_msr(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) msrs->secondary_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) msrs->secondary_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) case MSR_IA32_VMX_EPT_VPID_CAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) *pdata = msrs->ept_caps |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) ((u64)msrs->vpid_caps << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) case MSR_IA32_VMX_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) *pdata = msrs->vmfunc_controls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * Copy the writable VMCS shadow fields back to the VMCS12, in case they have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) * been modified by the L1 guest. Note, "writable" in this context means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) * VM-exit information fields (which are actually writable if the vCPU is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * configured to support "VMWRITE to any supported field in the VMCS").
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) struct shadow_vmcs_field field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) if (WARN_ON(!shadow_vmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) vmcs_load(shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) for (i = 0; i < max_shadow_read_write_fields; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) field = shadow_read_write_fields[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) val = __vmcs_readl(field.encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) vmcs12_write_any(vmcs12, field.encoding, field.offset, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) vmcs_clear(shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) vmcs_load(vmx->loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) const struct shadow_vmcs_field *fields[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) shadow_read_write_fields,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) shadow_read_only_fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) const int max_fields[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) max_shadow_read_write_fields,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) max_shadow_read_only_fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) struct shadow_vmcs_field field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) int i, q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) if (WARN_ON(!shadow_vmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) vmcs_load(shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) for (q = 0; q < ARRAY_SIZE(fields); q++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) for (i = 0; i < max_fields[q]; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) field = fields[q][i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) val = vmcs12_read_any(vmcs12, field.encoding,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) field.offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) __vmcs_writel(field.encoding, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) vmcs_clear(shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) vmcs_load(vmx->loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) vmcs12->tpr_threshold = evmcs->tpr_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) vmcs12->guest_rip = evmcs->guest_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) vmcs12->guest_rsp = evmcs->guest_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) vmcs12->guest_rflags = evmcs->guest_rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) vmcs12->guest_interruptibility_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) evmcs->guest_interruptibility_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) vmcs12->cpu_based_vm_exec_control =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) evmcs->cpu_based_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) vmcs12->exception_bitmap = evmcs->exception_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) vmcs12->vm_entry_controls = evmcs->vm_entry_controls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) vmcs12->vm_entry_intr_info_field =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) evmcs->vm_entry_intr_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) vmcs12->vm_entry_exception_error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) evmcs->vm_entry_exception_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) vmcs12->vm_entry_instruction_len =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) evmcs->vm_entry_instruction_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) vmcs12->host_ia32_pat = evmcs->host_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) vmcs12->host_ia32_efer = evmcs->host_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) vmcs12->host_cr0 = evmcs->host_cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) vmcs12->host_cr3 = evmcs->host_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) vmcs12->host_cr4 = evmcs->host_cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) vmcs12->host_rip = evmcs->host_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) vmcs12->host_es_selector = evmcs->host_es_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) vmcs12->host_cs_selector = evmcs->host_cs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) vmcs12->host_ss_selector = evmcs->host_ss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) vmcs12->host_ds_selector = evmcs->host_ds_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) vmcs12->host_fs_selector = evmcs->host_fs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) vmcs12->host_gs_selector = evmcs->host_gs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) vmcs12->host_tr_selector = evmcs->host_tr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) vmcs12->pin_based_vm_exec_control =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) evmcs->pin_based_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) vmcs12->vm_exit_controls = evmcs->vm_exit_controls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) vmcs12->secondary_vm_exec_control =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) evmcs->secondary_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) vmcs12->io_bitmap_a = evmcs->io_bitmap_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) vmcs12->io_bitmap_b = evmcs->io_bitmap_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) vmcs12->msr_bitmap = evmcs->msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) vmcs12->guest_es_base = evmcs->guest_es_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) vmcs12->guest_cs_base = evmcs->guest_cs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) vmcs12->guest_ss_base = evmcs->guest_ss_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) vmcs12->guest_ds_base = evmcs->guest_ds_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) vmcs12->guest_fs_base = evmcs->guest_fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) vmcs12->guest_gs_base = evmcs->guest_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) vmcs12->guest_tr_base = evmcs->guest_tr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) vmcs12->guest_idtr_base = evmcs->guest_idtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) vmcs12->guest_es_limit = evmcs->guest_es_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) vmcs12->guest_cs_limit = evmcs->guest_cs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) vmcs12->guest_ss_limit = evmcs->guest_ss_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) vmcs12->guest_ds_limit = evmcs->guest_ds_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) vmcs12->guest_fs_limit = evmcs->guest_fs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) vmcs12->guest_gs_limit = evmcs->guest_gs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) vmcs12->guest_tr_limit = evmcs->guest_tr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) vmcs12->guest_es_selector = evmcs->guest_es_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) vmcs12->guest_cs_selector = evmcs->guest_cs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) vmcs12->guest_ss_selector = evmcs->guest_ss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) vmcs12->guest_ds_selector = evmcs->guest_ds_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) vmcs12->guest_fs_selector = evmcs->guest_fs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) vmcs12->guest_gs_selector = evmcs->guest_gs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) vmcs12->guest_tr_selector = evmcs->guest_tr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) vmcs12->tsc_offset = evmcs->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) vmcs12->guest_cr0 = evmcs->guest_cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) vmcs12->guest_cr3 = evmcs->guest_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) vmcs12->guest_cr4 = evmcs->guest_cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) vmcs12->guest_dr7 = evmcs->guest_dr7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) vmcs12->host_fs_base = evmcs->host_fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) vmcs12->host_gs_base = evmcs->host_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) vmcs12->host_tr_base = evmcs->host_tr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) vmcs12->host_gdtr_base = evmcs->host_gdtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) vmcs12->host_idtr_base = evmcs->host_idtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) vmcs12->host_rsp = evmcs->host_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) vmcs12->ept_pointer = evmcs->ept_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) vmcs12->virtual_processor_id = evmcs->virtual_processor_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) if (unlikely(!(evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) vmcs12->guest_pdptr0 = evmcs->guest_pdptr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) vmcs12->guest_pdptr1 = evmcs->guest_pdptr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) vmcs12->guest_pdptr2 = evmcs->guest_pdptr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) vmcs12->guest_pdptr3 = evmcs->guest_pdptr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) vmcs12->guest_pending_dbg_exceptions =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) evmcs->guest_pending_dbg_exceptions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) vmcs12->guest_activity_state = evmcs->guest_activity_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) * Not used?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) * vmcs12->page_fault_error_code_mask =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) * evmcs->page_fault_error_code_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * vmcs12->page_fault_error_code_match =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * evmcs->page_fault_error_code_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * vmcs12->cr3_target_count = evmcs->cr3_target_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) * Read only fields:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) * vmcs12->guest_physical_address = evmcs->guest_physical_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) * vmcs12->vm_instruction_error = evmcs->vm_instruction_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) * vmcs12->vm_exit_reason = evmcs->vm_exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) * vmcs12->exit_qualification = evmcs->exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) * vmcs12->guest_linear_address = evmcs->guest_linear_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) * Not present in struct vmcs12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) * Should not be changed by KVM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) * evmcs->host_es_selector = vmcs12->host_es_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) * evmcs->host_cs_selector = vmcs12->host_cs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) * evmcs->host_ss_selector = vmcs12->host_ss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) * evmcs->host_ds_selector = vmcs12->host_ds_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) * evmcs->host_fs_selector = vmcs12->host_fs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) * evmcs->host_gs_selector = vmcs12->host_gs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) * evmcs->host_tr_selector = vmcs12->host_tr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) * evmcs->host_ia32_pat = vmcs12->host_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) * evmcs->host_ia32_efer = vmcs12->host_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) * evmcs->host_cr0 = vmcs12->host_cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) * evmcs->host_cr3 = vmcs12->host_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) * evmcs->host_cr4 = vmcs12->host_cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) * evmcs->host_rip = vmcs12->host_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) * evmcs->host_fs_base = vmcs12->host_fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) * evmcs->host_gs_base = vmcs12->host_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) * evmcs->host_tr_base = vmcs12->host_tr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * evmcs->host_gdtr_base = vmcs12->host_gdtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) * evmcs->host_idtr_base = vmcs12->host_idtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) * evmcs->host_rsp = vmcs12->host_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) * sync_vmcs02_to_vmcs12() doesn't read these:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) * evmcs->io_bitmap_a = vmcs12->io_bitmap_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) * evmcs->io_bitmap_b = vmcs12->io_bitmap_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) * evmcs->msr_bitmap = vmcs12->msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) * evmcs->ept_pointer = vmcs12->ept_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) * evmcs->tpr_threshold = vmcs12->tpr_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) * evmcs->virtual_processor_id = vmcs12->virtual_processor_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) * evmcs->exception_bitmap = vmcs12->exception_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) * evmcs->vm_exit_controls = vmcs12->vm_exit_controls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) * evmcs->page_fault_error_code_mask =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * vmcs12->page_fault_error_code_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) * evmcs->page_fault_error_code_match =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) * vmcs12->page_fault_error_code_match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) * evmcs->cr3_target_count = vmcs12->cr3_target_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) * evmcs->tsc_offset = vmcs12->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) * Not present in struct vmcs12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) evmcs->guest_es_selector = vmcs12->guest_es_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) evmcs->guest_cs_selector = vmcs12->guest_cs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) evmcs->guest_ss_selector = vmcs12->guest_ss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) evmcs->guest_ds_selector = vmcs12->guest_ds_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) evmcs->guest_fs_selector = vmcs12->guest_fs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) evmcs->guest_gs_selector = vmcs12->guest_gs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) evmcs->guest_tr_selector = vmcs12->guest_tr_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) evmcs->guest_es_limit = vmcs12->guest_es_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) evmcs->guest_cs_limit = vmcs12->guest_cs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) evmcs->guest_ss_limit = vmcs12->guest_ss_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) evmcs->guest_ds_limit = vmcs12->guest_ds_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) evmcs->guest_fs_limit = vmcs12->guest_fs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) evmcs->guest_gs_limit = vmcs12->guest_gs_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) evmcs->guest_tr_limit = vmcs12->guest_tr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) evmcs->guest_es_base = vmcs12->guest_es_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) evmcs->guest_cs_base = vmcs12->guest_cs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) evmcs->guest_ss_base = vmcs12->guest_ss_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) evmcs->guest_ds_base = vmcs12->guest_ds_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) evmcs->guest_fs_base = vmcs12->guest_fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) evmcs->guest_gs_base = vmcs12->guest_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) evmcs->guest_tr_base = vmcs12->guest_tr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) evmcs->guest_idtr_base = vmcs12->guest_idtr_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) evmcs->guest_pdptr0 = vmcs12->guest_pdptr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) evmcs->guest_pdptr1 = vmcs12->guest_pdptr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) evmcs->guest_pdptr2 = vmcs12->guest_pdptr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) evmcs->guest_pdptr3 = vmcs12->guest_pdptr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) evmcs->guest_pending_dbg_exceptions =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) vmcs12->guest_pending_dbg_exceptions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) evmcs->guest_activity_state = vmcs12->guest_activity_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) evmcs->guest_cr0 = vmcs12->guest_cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) evmcs->guest_cr3 = vmcs12->guest_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) evmcs->guest_cr4 = vmcs12->guest_cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) evmcs->guest_dr7 = vmcs12->guest_dr7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) evmcs->guest_physical_address = vmcs12->guest_physical_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) evmcs->vm_instruction_error = vmcs12->vm_instruction_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) evmcs->vm_exit_reason = vmcs12->vm_exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) evmcs->exit_qualification = vmcs12->exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) evmcs->guest_linear_address = vmcs12->guest_linear_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) evmcs->guest_rsp = vmcs12->guest_rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) evmcs->guest_rflags = vmcs12->guest_rflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) evmcs->guest_interruptibility_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) vmcs12->guest_interruptibility_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) evmcs->vm_entry_controls = vmcs12->vm_entry_controls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) evmcs->vm_entry_exception_error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) vmcs12->vm_entry_exception_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) evmcs->guest_rip = vmcs12->guest_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) * This is an equivalent of the nested hypervisor executing the vmptrld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) * instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) struct kvm_vcpu *vcpu, bool from_launch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) bool evmcs_gpa_changed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) u64 evmcs_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) if (likely(!vmx->nested.enlightened_vmcs_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) return EVMPTRLD_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) return EVMPTRLD_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) if (unlikely(!vmx->nested.hv_evmcs ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) if (!vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) vmx->nested.current_vmptr = -1ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) nested_release_evmcs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) &vmx->nested.hv_evmcs_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) return EVMPTRLD_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) * Currently, KVM only supports eVMCS version 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) * (== KVM_EVMCS_VERSION) and thus we expect guest to set this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) * value to first u32 field of eVMCS which should specify eVMCS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) * VersionNumber.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) * Guest should be aware of supported eVMCS versions by host by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) * expected to set this CPUID leaf according to the value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) * returned in vmcs_version from nested_enable_evmcs().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) * However, it turns out that Microsoft Hyper-V fails to comply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) * to their own invented interface: When Hyper-V use eVMCS, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) * just sets first u32 field of eVMCS to revision_id specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) * which is one of the supported versions specified in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * CPUID.0x4000000A.EAX[0:15].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * To overcome Hyper-V bug, we accept here either a supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * eVMCS version or VMCS12 revision_id as valid values for first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) * u32 field of eVMCS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) nested_release_evmcs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) return EVMPTRLD_VMFAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) vmx->nested.dirty_vmcs12 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) evmcs_gpa_changed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) * Unlike normal vmcs12, enlightened vmcs12 is not fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * reloaded from guest's memory (read only fields, fields not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) * present in struct hv_enlightened_vmcs, ...). Make sure there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) * are no leftovers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) if (from_launch) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) memset(vmcs12, 0, sizeof(*vmcs12));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) vmcs12->hdr.revision_id = VMCS12_REVISION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) * Clean fields data can't be used on VMLAUNCH and when we switch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) * between different L2 guests as KVM keeps a single VMCS12 per L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) if (from_launch || evmcs_gpa_changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) vmx->nested.hv_evmcs->hv_clean_fields &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) return EVMPTRLD_SUCCEEDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) if (vmx->nested.hv_evmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) copy_vmcs12_to_enlightened(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) /* All fields are clean */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) vmx->nested.hv_evmcs->hv_clean_fields |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) copy_vmcs12_to_shadow(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) vmx->nested.need_vmcs12_to_shadow_sync = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) struct vcpu_vmx *vmx =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) container_of(timer, struct vcpu_vmx, nested.preemption_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) vmx->nested.preemption_timer_expired = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) kvm_vcpu_kick(&vmx->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) if (!vmx->nested.has_preemption_timer_deadline) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) vmx->nested.preemption_timer_deadline =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) vmcs12->vmx_preemption_timer_value + l1_scaled_tsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) vmx->nested.has_preemption_timer_deadline = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) return vmx->nested.preemption_timer_deadline - l1_scaled_tsc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) u64 preemption_timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) * A timer value of zero is architecturally guaranteed to cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) * a VMExit prior to executing any instructions in the guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) if (preemption_timeout == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) if (vcpu->arch.virtual_tsc_khz == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) preemption_timeout *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) hrtimer_start(&vmx->nested.preemption_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) ktime_add_ns(ktime_get(), preemption_timeout),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) HRTIMER_MODE_ABS_PINNED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) if (vmx->nested.nested_run_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) return vmcs12->guest_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) * If vmcs02 hasn't been initialized, set the constant vmcs02 state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) * according to L0's settings (vmcs12 is irrelevant here). Host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) * fields that come from L0 and are not constant, e.g. HOST_CR3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) * will be set as needed prior to VMLAUNCH/VMRESUME.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) if (vmx->nested.vmcs02_initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) vmx->nested.vmcs02_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) * We don't care what the EPTP value is we just need to guarantee
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) * it's valid so we don't get a false positive when doing early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) * consistency checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) if (enable_ept && nested_early_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) vmcs_write64(EPT_POINTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) construct_eptp(&vmx->vcpu, 0, PT64_ROOT_4LEVEL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) /* All VMFUNCs are currently emulated through L0 vmexits. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) if (cpu_has_vmx_vmfunc())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) vmcs_write64(VM_FUNCTION_CONTROL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) if (cpu_has_vmx_posted_intr())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) * The PML address never changes, so it is constant in vmcs02.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) * Conceptually we want to copy the PML index from vmcs01 here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) * and then back to vmcs01 on nested vmexit. But since we flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) * the log and reset GUEST_PML_INDEX on each vmexit, the PML
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) * index is also effectively constant in vmcs02.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) if (enable_pml) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) if (cpu_has_vmx_encls_vmexit())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) * Set the MSR load/store lists to match L0's settings. Only the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) * addresses are constant (for vmcs02), the counts can change based
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) * on L2's behavior, e.g. switching to/from long mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) vmx_set_constant_host_state(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) prepare_vmcs02_constant_state(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) vmcs_write64(VMCS_LINK_POINTER, -1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) if (enable_vpid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) u32 exec_control, vmcs12_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) prepare_vmcs02_early_rare(vmx, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) * PIN CONTROLS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) exec_control = vmx_pin_based_exec_ctrl(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) exec_control |= (vmcs12->pin_based_vm_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) ~PIN_BASED_VMX_PREEMPTION_TIMER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) /* Posted interrupts setting is only taken from vmcs12. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) vmx->nested.pi_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) if (nested_cpu_has_posted_intr(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) exec_control &= ~PIN_BASED_POSTED_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) pin_controls_set(vmx, exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) * EXEC CONTROLS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) exec_control = vmx_exec_control(vmx); /* L0's desires */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) exec_control &= ~CPU_BASED_TPR_SHADOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) exec_control |= vmcs12->cpu_based_vm_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) vmx->nested.l1_tpr_threshold = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) if (exec_control & CPU_BASED_TPR_SHADOW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) exec_control |= CPU_BASED_CR8_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) CPU_BASED_CR8_STORE_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) * for I/O port accesses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) exec_control |= CPU_BASED_UNCOND_IO_EXITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) * This bit will be computed in nested_get_vmcs12_pages, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) * we do not have access to L1's MSR bitmap yet. For now, keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) * the same bit as before, hoping to avoid multiple VMWRITEs that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) * only set/clear this bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) exec_controls_set(vmx, exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) * SECONDARY EXEC CONTROLS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) if (cpu_has_secondary_exec_ctrls()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) exec_control = vmx->secondary_exec_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) /* Take the following fields only from vmcs12 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) SECONDARY_EXEC_ENABLE_INVPCID |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) SECONDARY_EXEC_ENABLE_RDTSCP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) SECONDARY_EXEC_XSAVES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) SECONDARY_EXEC_ENABLE_VMFUNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) if (nested_cpu_has(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) ~SECONDARY_EXEC_ENABLE_PML;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) exec_control |= vmcs12_exec_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) /* VMCS shadowing for L2 is emulated for now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) * will not have to rewrite the controls just for this bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) (vmcs12->guest_cr4 & X86_CR4_UMIP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) exec_control |= SECONDARY_EXEC_DESC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) vmcs_write16(GUEST_INTR_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) vmcs12->guest_intr_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) secondary_exec_controls_set(vmx, exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) * ENTRY CONTROLS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) * on the related bits (if supported by the CPU) in the hope that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) * we can avoid VMWrites during vmx_set_efer().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) if (cpu_has_load_ia32_efer()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) if (guest_efer & EFER_LMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) exec_control |= VM_ENTRY_IA32E_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) if (guest_efer != host_efer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) exec_control |= VM_ENTRY_LOAD_IA32_EFER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) vm_entry_controls_set(vmx, exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) * EXIT CONTROLS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) * L2->L1 exit controls are emulated - the hardware exit is to L0 so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) * bits may be modified by vmx_set_efer() in prepare_vmcs02().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) exec_control = vmx_vmexit_ctrl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) if (cpu_has_load_ia32_efer() && guest_efer != host_efer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) exec_control |= VM_EXIT_LOAD_IA32_EFER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) vm_exit_controls_set(vmx, exec_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) * Interrupt/Exception Fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (vmx->nested.nested_run_pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) vmcs12->vm_entry_intr_info_field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) vmcs12->vm_entry_exception_error_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) vmcs12->vm_entry_instruction_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) vmcs12->guest_interruptibility_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) vmx->loaded_vmcs->nmi_known_unmasked =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) vmx->segment_cache.bitmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) vmcs12->guest_pending_dbg_exceptions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) * L1 may access the L2's PDPTR, so save them to construct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) * vmcs12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) if (nested_cpu_has_xsaves(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) * Whether page-faults are trapped is determined by a combination of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) * doesn't care about page faults then we should set all of these to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) * L1's desires. However, if L0 does care about (some) page faults, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) * is not easy (if at all possible?) to merge L0 and L1's desires, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) * simply ask to exit on each and every L2 page fault. This is done by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) * setting MASK=MATCH=0 and (see below) EB.PF=1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) * Note that below we don't need special code to set EB.PF beyond the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) * !enable_ept, EB.PF is 1, so the "or" will always be 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) if (vmx_need_pf_intercept(&vmx->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) * TODO: if both L0 and L1 need the same MASK and MATCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) * go ahead and use it?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) if (cpu_has_vmx_apicv()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) * Make sure the msr_autostore list is up to date before we set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) * count in the vmcs02.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) set_cr4_guest_host_mask(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) * guest in a way that will both be appropriate to L1's requests, and our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) * needs. In addition to modifying the active vmcs (which is vmcs02), this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) * function also has additional necessary side-effects, like setting various
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) * vcpu->arch fields.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) * Returns 0 on success, 1 on failure. Invalid state exit qualification code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) * is assigned to entry_failure_code on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) enum vm_entry_failure_code *entry_failure_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) bool load_guest_pdptrs_vmcs12 = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) if (vmx->nested.dirty_vmcs12 || hv_evmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) prepare_vmcs02_rare(vmx, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) vmx->nested.dirty_vmcs12 = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) load_guest_pdptrs_vmcs12 = !hv_evmcs ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) !(hv_evmcs->hv_clean_fields &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) if (vmx->nested.nested_run_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) vmx_set_rflags(vcpu, vmcs12->guest_rflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) * bitwise-or of what L1 wants to trap for L2, and what we want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) * trap. Note that CR0.TS also needs updating - we do this later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) update_exception_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) if (vmx->nested.nested_run_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) vcpu->arch.pat = vmcs12->guest_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) if (kvm_has_tsc_control)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) decache_tsc_multiplier(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) nested_vmx_transition_tlb_flush(vcpu, vmcs12, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) if (nested_cpu_has_ept(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) nested_ept_init_mmu_context(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) * bits which we consider mandatory enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) * The CR0_READ_SHADOW is what L2 should have expected to read given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) * the specifications by L1; It's not enough to take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) * have more bits than L1 expected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) vmx_set_cr0(vcpu, vmcs12->guest_cr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) vmx_set_cr4(vcpu, vmcs12->guest_cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) vmx_set_efer(vcpu, vcpu->arch.efer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) * Guest state is invalid and unrestricted guest is disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) * which means L1 attempted VMEntry to L2 with invalid state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) * Fail the VMEntry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) if (CC(!vmx_guest_state_valid(vcpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) *entry_failure_code = ENTRY_FAIL_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) /* Shadow page tables on either EPT or shadow page tables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) entry_failure_code))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) * on nested VM-Exit, which can occur without actually running L2 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) * transition to HLT instead of running L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) vmcs_writel(GUEST_CR3, vmcs12->guest_cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) is_pae_paging(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) vmcs12->guest_ia32_perf_global_ctrl))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) *entry_failure_code = ENTRY_FAIL_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) kvm_rsp_write(vcpu, vmcs12->guest_rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) kvm_rip_write(vcpu, vmcs12->guest_rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) nested_cpu_has_virtual_nmis(vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) int maxphyaddr = cpuid_maxphyaddr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) /* Check for memory type validity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) switch (new_eptp & VMX_EPTP_MT_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) case VMX_EPTP_MT_UC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) case VMX_EPTP_MT_WB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) /* Page-walk levels validity. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) switch (new_eptp & VMX_EPTP_PWL_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) case VMX_EPTP_PWL_5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) case VMX_EPTP_PWL_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) /* Reserved bits should not be set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) if (CC(new_eptp >> maxphyaddr || ((new_eptp >> 7) & 0x1f)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) /* AD, if set, should be supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) * Checks related to VM-Execution Control Fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) vmx->nested.msrs.pinbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) vmx->nested.msrs.pinbased_ctls_high)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) vmx->nested.msrs.procbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) vmx->nested.msrs.procbased_ctls_high)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) vmx->nested.msrs.secondary_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) vmx->nested.msrs.secondary_ctls_high)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) nested_vmx_check_apic_access_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) nested_vmx_check_apicv_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) nested_vmx_check_nmi_controls(vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) nested_vmx_check_pml_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) if (!nested_cpu_has_preemption_timer(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) nested_cpu_has_save_preemption_timer(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) if (nested_cpu_has_ept(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) if (nested_cpu_has_vmfunc(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) if (CC(vmcs12->vm_function_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) ~vmx->nested.msrs.vmfunc_controls))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) if (nested_cpu_has_eptp_switching(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) if (CC(!nested_cpu_has_ept(vmcs12)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) * Checks related to VM-Exit Control Fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) vmx->nested.msrs.exit_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) vmx->nested.msrs.exit_ctls_high)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) * Checks related to VM-Entry Control Fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) vmx->nested.msrs.entry_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) vmx->nested.msrs.entry_ctls_high)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) * From the Intel SDM, volume 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) * Fields relevant to VM-entry event injection must be set properly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) * These fields are the VM-entry interruption-information field, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) * VM-entry exception error code, and the VM-entry instruction length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) u32 intr_info = vmcs12->vm_entry_intr_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) bool should_have_error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) bool urg = nested_cpu_has2(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) SECONDARY_EXEC_UNRESTRICTED_GUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) /* VM-entry interruption-info field: interruption type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) if (CC(intr_type == INTR_TYPE_RESERVED) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) CC(intr_type == INTR_TYPE_OTHER_EVENT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) !nested_cpu_supports_monitor_trap_flag(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) /* VM-entry interruption-info field: vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) /* VM-entry interruption-info field: deliver error code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) should_have_error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) x86_exception_has_error_code(vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) if (CC(has_error_code != should_have_error_code))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) /* VM-entry exception error code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) if (CC(has_error_code &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) vmcs12->vm_entry_exception_error_code & GENMASK(31, 16)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) /* VM-entry interruption-info field: reserved bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) /* VM-entry instruction length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) switch (intr_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) case INTR_TYPE_SOFT_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) case INTR_TYPE_SOFT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) case INTR_TYPE_PRIV_SW_EXCEPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) if (CC(vmcs12->vm_entry_instruction_len > 15) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) CC(vmcs12->vm_entry_instruction_len == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) CC(!nested_cpu_has_zero_length_injection(vcpu))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) if (nested_check_vm_execution_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) nested_check_vm_exit_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) nested_check_vm_entry_controls(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) return nested_evmcs_check_controls(vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) !!(vcpu->arch.efer & EFER_LMA)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) bool ia32e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) vmcs12->host_ia32_perf_global_ctrl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) ia32e = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) if (ia32e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) CC(vmcs12->host_cr4 & X86_CR4_PCIDE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) CC((vmcs12->host_rip) >> 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) CC(vmcs12->host_cs_selector == 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) CC(vmcs12->host_tr_selector == 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) CC(vmcs12->host_ss_selector == 0 && !ia32e))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) CC(is_noncanonical_address(vmcs12->host_rip, vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) * If the load IA32_EFER VM-exit control is 1, bits reserved in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) * IA32_EFER MSR must be 0 in the field for that register. In addition,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) * the values of the LMA and LME bits in the field must each be that of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) * the host address-space size VM-exit control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) struct vmcs12 *shadow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) struct kvm_host_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) if (vmcs12->vmcs_link_pointer == -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) shadow = map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) if (CC(shadow->hdr.revision_id != VMCS12_REVISION) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) kvm_vcpu_unmap(vcpu, &map, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) * Checks related to Guest Non-register State
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) enum vm_entry_failure_code *entry_failure_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) bool ia32e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) *entry_failure_code = ENTRY_FAIL_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) CC(!kvm_dr7_valid(vmcs12->guest_dr7)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) vmcs12->guest_ia32_perf_global_ctrl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) * If the load IA32_EFER VM-entry control is 1, the following checks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) * are performed on the field for the IA32_EFER MSR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) * - Bits reserved in the IA32_EFER MSR must be 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) * the IA-32e mode guest VM-exit control. It must also be identical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) * CR0.PG) is 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) if (to_vmx(vcpu)->nested.nested_run_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) if (nested_check_guest_non_reg_state(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) unsigned long cr3, cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) bool vm_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) if (!nested_early_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) if (vmx->msr_autoload.host.nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) if (vmx->msr_autoload.guest.nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) vmx_prepare_switch_to_guest(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) * be written (by prepare_vmcs02()) before the "real" VMEnter, i.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) * there is no need to preserve other bits or save/restore the field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) vmcs_writel(GUEST_RFLAGS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) cr3 = __get_current_cr3_fast();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) vmcs_writel(HOST_CR3, cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) vmx->loaded_vmcs->host_state.cr3 = cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) cr4 = cr4_read_shadow();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) vmcs_writel(HOST_CR4, cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) vmx->loaded_vmcs->host_state.cr4 = cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) asm(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) "je 1f \n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) "1: \n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) /* Check if vmlaunch or vmresume is needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) "cmpb $0, %c[launched](%[loaded_vmcs])\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) "call vmx_vmenter\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) CC_SET(be)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) : [HOST_RSP]"r"((unsigned long)HOST_RSP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) [loaded_vmcs]"r"(vmx->loaded_vmcs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) [launched]"i"(offsetof(struct loaded_vmcs, launched)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) [wordsize]"i"(sizeof(ulong))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) : "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) if (vmx->msr_autoload.host.nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) if (vmx->msr_autoload.guest.nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) if (vm_fail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) trace_kvm_nested_vmenter_failed(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) "early hardware check VM-instruction error: ", error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) * VMExit clears RFLAGS.IF and DR7, even on a consistency check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) if (hw_breakpoint_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) set_debugreg(__this_cpu_read(cpu_dr7), 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) * A non-failing VMEntry means we somehow entered guest mode with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) * an illegal RIP, and that's just the tip of the iceberg. There
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) * is no telling what memory has been modified or what state has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) * been exposed to unknown code. Hitting this all but guarantees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) * a (very critical) hardware issue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) VMX_EXIT_REASONS_FAILED_VMENTRY));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) * hv_evmcs may end up being not mapped after migration (when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) * L2 was running), map it here to make sure vmcs12 changes are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) * properly reflected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) enum nested_evmptrld_status evmptrld_status =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) nested_vmx_handle_enlightened_vmptrld(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) if (evmptrld_status == EVMPTRLD_VMFAIL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) evmptrld_status == EVMPTRLD_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) struct kvm_host_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) u64 hpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) * Translate L1 physical address to host physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) * address for vmcs02. Keep the page pinned, so this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) * physical address remains valid. We keep a reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) * to it so we can release it later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) if (vmx->nested.apic_access_page) { /* shouldn't happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) kvm_release_page_clean(vmx->nested.apic_access_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) vmx->nested.apic_access_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) if (!is_error_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) vmx->nested.apic_access_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) hpa = page_to_phys(vmx->nested.apic_access_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) vmcs_write64(APIC_ACCESS_ADDR, hpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) vcpu->run->internal.suberror =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) KVM_INTERNAL_ERROR_EMULATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) vcpu->run->internal.ndata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) map = &vmx->nested.virtual_apic_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * The processor will never use the TPR shadow, simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) * clear the bit from the execution control. Such a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) * configuration is useless, but it happens in tests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) * For any other configuration, failing the vm entry is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) * _not_ what the processor does but it's basically the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) * only possibility we have.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) * force VM-Entry to fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) if (nested_cpu_has_posted_intr(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) map = &vmx->nested.pi_desc_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) vmx->nested.pi_desc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) (struct pi_desc *)(((void *)map->hva) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) offset_in_page(vmcs12->posted_intr_desc_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) vmcs_write64(POSTED_INTR_DESC_ADDR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) if (!nested_get_evmcs_page(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) pr_debug_ratelimited("%s: enlightened vmptrld failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) vcpu->run->internal.suberror =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) KVM_INTERNAL_ERROR_EMULATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) vcpu->run->internal.ndata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) struct vmcs12 *vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) gpa_t dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) if (WARN_ON_ONCE(!is_guest_mode(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) if (WARN_ON_ONCE(vmx->nested.pml_full))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) * Check if PML is enabled for the nested guest. Whether eptp bit 6 is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) * set is already checked as part of A/D emulation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) if (!nested_cpu_has_pml(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) vmx->nested.pml_full = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) gpa &= ~0xFFFull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) offset_in_page(dst), sizeof(gpa)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) vmcs12->guest_pml_index--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) * Intel's VMX Instruction Reference specifies a common set of prerequisites
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) * for running VMX instructions (except VMXON, whose prerequisites are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) * slightly different). It also specifies what exception to inject otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) * Note that many of these exceptions have priority over VM exits, so they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) * don't have to be checked again here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) if (!to_vmx(vcpu)->nested.vmxon) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) if (vmx_get_cpl(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) kvm_inject_gp(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) u8 rvi = vmx_get_rvi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) return ((rvi & 0xf0) > (vppr & 0xf0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) struct vmcs12 *vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) * If from_vmentry is false, this is being called from state restore (either RSM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) * NVMX_VMENTRY_VMFAIL: Consistency check VMFail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) * NVMX_VMENTRY_VMEXIT: Consistency check VMExit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) bool from_vmentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) enum vm_entry_failure_code entry_failure_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) bool evaluate_pending_interrupts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) union vmx_exit_reason exit_reason = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) .basic = EXIT_REASON_INVALID_STATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) .failed_vmentry = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) u32 failed_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) kvm_vcpu_flush_tlb_current(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) evaluate_pending_interrupts = exec_controls_get(vmx) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) if (kvm_mpx_supported() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) * nested early checks are disabled. In the event of a "late" VM-Fail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) * software model to the pre-VMEntry host state. When EPT is disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3. Stuffing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) * the correct value. Smashing vmcs01.GUEST_CR3 is safe because nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) * guaranteed to be overwritten with a shadow CR3 prior to re-entering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) * L1. Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) * path would need to manually save/restore vmcs01.GUEST_CR3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) if (!enable_ept && !nested_early_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) prepare_vmcs02_early(vmx, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) if (from_vmentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) if (unlikely(!nested_get_vmcs12_pages(vcpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) vmx_switch_vmcs(vcpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) if (nested_vmx_check_vmentry_hw(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) vmx_switch_vmcs(vcpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) return NVMX_VMENTRY_VMFAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) if (nested_vmx_check_guest_state(vcpu, vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) &entry_failure_code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) exit_reason.basic = EXIT_REASON_INVALID_STATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) vmcs12->exit_qualification = entry_failure_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) goto vmentry_fail_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) enter_guest_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) vcpu->arch.tsc_offset += vmcs12->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) exit_reason.basic = EXIT_REASON_INVALID_STATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) vmcs12->exit_qualification = entry_failure_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) goto vmentry_fail_vmexit_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) if (from_vmentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) failed_index = nested_vmx_load_msr(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) vmcs12->vm_entry_msr_load_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) vmcs12->vm_entry_msr_load_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) if (failed_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) vmcs12->exit_qualification = failed_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) goto vmentry_fail_vmexit_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) * The MMU is not initialized to point at the right entities yet and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) * "get pages" would need to read data from the guest (i.e. we will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) * need to perform gpa to hpa translation). Request a call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) * to nested_get_vmcs12_pages before the next VM-entry. The MSRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) * have already been set at vmentry time and should not be reset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) * If L1 had a pending IRQ/NMI until it executed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) * VMLAUNCH/VMRESUME which wasn't delivered because it was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) * disallowed (e.g. interrupts disabled), L0 needs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) * evaluate if this pending event should cause an exit from L2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) * to L1 or delivered directly to L2 (e.g. In case L1 don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) * intercept EXTERNAL_INTERRUPT).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) * Usually this would be handled by the processor noticing an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) * IRQ/NMI window request, or checking RVI during evaluation of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) * pending virtual interrupts. However, this setting was done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) * on VMCS01 and now VMCS02 is active instead. Thus, we force L0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) * to perform pending event evaluation by requesting a KVM_REQ_EVENT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) if (unlikely(evaluate_pending_interrupts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) * Do not start the preemption timer hrtimer until after we know
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) * we are successful, so that only nested_vmx_vmexit needs to cancel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) * the timer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) vmx->nested.preemption_timer_expired = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) if (nested_cpu_has_preemption_timer(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) u64 timer_value = vmx_calc_preemption_timer_value(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) vmx_start_preemption_timer(vcpu, timer_value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) * returned as far as L1 is concerned. It will only return (and set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) * the success flag) when L2 exits (see nested_vmx_vmexit()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) return NVMX_VMENTRY_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) * A failed consistency check that leads to a VMExit during L1's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) * VMEnter to L2 is a variation of a normal VMexit, as explained in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) * 26.7 "VM-entry failures during or after loading guest state".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) vmentry_fail_vmexit_guest_mode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) leave_guest_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) vmentry_fail_vmexit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) vmx_switch_vmcs(vcpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) if (!from_vmentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) return NVMX_VMENTRY_VMEXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) load_vmcs12_host_state(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) vmcs12->vm_exit_reason = exit_reason.full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) if (enable_shadow_vmcs || vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) vmx->nested.need_vmcs12_to_shadow_sync = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) return NVMX_VMENTRY_VMEXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) * for running an L2 nested guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) struct vmcs12 *vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) enum nvmx_vmentry_status status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) enum nested_evmptrld_status evmptrld_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) if (evmptrld_status == EVMPTRLD_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) } else if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) if (CC(!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) * that there *is* a valid VMCS pointer, RFLAGS.CF is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) * rather than RFLAGS.ZF, and no error number is stored to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) * VM-instruction error field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) if (CC(vmcs12->hdr.shadow_vmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) if (vmx->nested.hv_evmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) copy_enlightened_to_vmcs12(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) /* Enlightened VMCS doesn't have launch state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) vmcs12->launch_state = !launch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) } else if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) copy_shadow_to_vmcs12(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) * The nested entry process starts with enforcing various prerequisites
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) * on vmcs12 as required by the Intel SDM, and act appropriately when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) * they fail: As the SDM explains, some conditions should cause the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) * instruction to fail, while others will cause the instruction to seem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) * to succeed, but return an EXIT_REASON_INVALID_STATE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) * To speed up the normal (success) code path, we should avoid checking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) * for misconfigurations which will anyway be caught by the processor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) * when using the merged vmcs02.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) if (CC(vmcs12->launch_state == launch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) if (nested_vmx_check_controls(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) if (nested_vmx_check_address_space_size(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) if (nested_vmx_check_host_state(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) * We're finally done with prerequisite checking, and can start with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) * the nested entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) vmx->nested.nested_run_pending = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) vmx->nested.has_preemption_timer_deadline = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) status = nested_vmx_enter_non_root_mode(vcpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) if (unlikely(status != NVMX_VMENTRY_SUCCESS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) goto vmentry_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) /* Emulate processing of posted interrupts on VM-Enter. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) if (nested_cpu_has_posted_intr(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) kvm_apic_has_interrupt(vcpu) == vmx->nested.posted_intr_nv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) vmx->nested.pi_pending = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) kvm_apic_clear_irr(vcpu, vmx->nested.posted_intr_nv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) /* Hide L1D cache contents from the nested guest. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) vmx->vcpu.arch.l1tf_flush_l1d = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) * Must happen outside of nested_vmx_enter_non_root_mode() as it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) * also be used as part of restoring nVMX state for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) * snapshot restore (migration).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) * In this flow, it is assumed that vmcs12 cache was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) * trasferred as part of captured nVMX state and should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) * therefore not be read from guest memory (which may not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) * exist on destination host yet).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) nested_cache_shadow_vmcs12(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) * If we're entering a halted L2 vcpu and the L2 vcpu won't be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) * awakened by event injection or by an NMI-window VM-exit or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) * by an interrupt-window VM-exit, halt the vcpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_NMI_WINDOW_EXITING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_INTR_WINDOW_EXITING) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) vmx->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) return kvm_vcpu_halt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) vmentry_failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) vmx->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) if (status == NVMX_VMENTRY_VMEXIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) * This function returns the new value we should put in vmcs12.guest_cr0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) * didn't trap the bit, because if L1 did, so would L0).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) * been modified by L2, and L1 knows it. So just leave the old value of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) * isn't relevant, because if L0 traps this bit it can set it to anything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) * changed these bits, and therefore they need to be updated, but L0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) * didn't necessarily allow them to be changed in GUEST_CR0 - and rather
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) static inline unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) vcpu->arch.cr0_guest_owned_bits));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) static inline unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) vcpu->arch.cr4_guest_owned_bits));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) u32 idt_vectoring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) unsigned int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) if (vcpu->arch.exception.injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) nr = vcpu->arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) if (kvm_exception_is_soft(nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) vmcs12->vm_exit_instruction_len =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) vcpu->arch.event_exit_inst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) if (vcpu->arch.exception.has_error_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) vmcs12->idt_vectoring_error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) vcpu->arch.exception.error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) vmcs12->idt_vectoring_info_field = idt_vectoring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) } else if (vcpu->arch.nmi_injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) vmcs12->idt_vectoring_info_field =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) } else if (vcpu->arch.interrupt.injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) nr = vcpu->arch.interrupt.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) if (vcpu->arch.interrupt.soft) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) idt_vectoring |= INTR_TYPE_SOFT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) vmcs12->vm_entry_instruction_len =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) vcpu->arch.event_exit_inst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) idt_vectoring |= INTR_TYPE_EXT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) vmcs12->idt_vectoring_info_field = idt_vectoring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) gfn_t gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) * Don't need to mark the APIC access page dirty; it is never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) * written to by the CPU during APIC virtualization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) kvm_vcpu_mark_page_dirty(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) if (nested_cpu_has_posted_intr(vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) kvm_vcpu_mark_page_dirty(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) int max_irr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) void *vapic_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) u16 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) vmx->nested.pi_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) if (!pi_test_and_clear_on(vmx->nested.pi_desc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) if (max_irr != 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) vapic_page = vmx->nested.virtual_apic_map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) if (!vapic_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) vapic_page, &max_irr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) status = vmcs_read16(GUEST_INTR_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) if ((u8)max_irr > ((u8)status & 0xff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) status &= ~0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) status |= (u8)max_irr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) vmcs_write16(GUEST_INTR_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) nested_mark_vmcs12_pages_dirty(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) unsigned long exit_qual)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) unsigned int nr = vcpu->arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) u32 intr_info = nr | INTR_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) if (vcpu->arch.exception.has_error_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) intr_info |= INTR_INFO_DELIVER_CODE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) if (kvm_exception_is_soft(nr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) intr_info |= INTR_TYPE_SOFT_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) intr_info |= INTR_TYPE_HARD_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) vmx_get_nmi_mask(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) intr_info |= INTR_INFO_UNBLOCK_NMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) * Returns true if a debug trap is pending delivery.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) * In KVM, debug traps bear an exception payload. As such, the class of a #DB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) * exception may be inferred from the presence of an exception payload.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) return vcpu->arch.exception.pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) vcpu->arch.exception.nr == DB_VECTOR &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) vcpu->arch.exception.payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) * Certain VM-exits set the 'pending debug exceptions' field to indicate a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) * recognized #DB (data or single-step) that has yet to be delivered. Since KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) * represents these debug traps with a payload that is said to be compatible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) * with the 'pending debug exceptions' field, write the payload to the VMCS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) * field if a VM-exit is delivered before the debug trap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) if (vmx_pending_dbg_trap(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) vcpu->arch.exception.payload);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) to_vmx(vcpu)->nested.preemption_timer_expired;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) unsigned long exit_qual;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) bool block_nested_events =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) bool mtf_pending = vmx->nested.mtf_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) struct kvm_lapic *apic = vcpu->arch.apic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) * Clear the MTF state. If a higher priority VM-exit is delivered first,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) * this state is discarded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) if (!block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) vmx->nested.mtf_pending = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) if (lapic_in_kernel(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) test_bit(KVM_APIC_INIT, &apic->pending_events)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) nested_vmx_update_pending_dbg(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) clear_bit(KVM_APIC_INIT, &apic->pending_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) * Process any exceptions that are not debug traps before MTF.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) if (vcpu->arch.exception.pending && !vmx_pending_dbg_trap(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) if (!nested_vmx_check_exception(vcpu, &exit_qual))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) goto no_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) if (mtf_pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) nested_vmx_update_pending_dbg(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) if (vcpu->arch.exception.pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) if (!nested_vmx_check_exception(vcpu, &exit_qual))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) goto no_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) if (nested_vmx_preemption_timer_pending(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) if (vcpu->arch.smi_pending && !is_smm(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) goto no_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) if (!nested_exit_on_nmi(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) goto no_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) NMI_VECTOR | INTR_TYPE_NMI_INTR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) INTR_INFO_VALID_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) * The NMI-triggered VM exit counts as injection:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) * clear this one and block further NMIs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) vcpu->arch.nmi_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) vmx_set_nmi_mask(vcpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) if (!nested_exit_on_intr(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) goto no_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) no_vmexit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) vmx_complete_nested_posted_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) ktime_t remaining =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) u64 value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) if (ktime_to_ns(remaining) <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) do_div(value, 1000000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) static bool is_vmcs12_ext_field(unsigned long field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) switch (field) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) case GUEST_ES_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) case GUEST_CS_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) case GUEST_SS_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) case GUEST_DS_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) case GUEST_FS_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) case GUEST_GS_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) case GUEST_LDTR_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) case GUEST_TR_SELECTOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) case GUEST_ES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) case GUEST_CS_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) case GUEST_SS_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) case GUEST_DS_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) case GUEST_FS_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) case GUEST_GS_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) case GUEST_LDTR_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) case GUEST_TR_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) case GUEST_GDTR_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) case GUEST_IDTR_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) case GUEST_ES_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) case GUEST_DS_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) case GUEST_FS_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) case GUEST_GS_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) case GUEST_LDTR_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) case GUEST_TR_AR_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) case GUEST_ES_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) case GUEST_CS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) case GUEST_SS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) case GUEST_DS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) case GUEST_FS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) case GUEST_GS_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) case GUEST_LDTR_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) case GUEST_TR_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) case GUEST_GDTR_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) case GUEST_IDTR_BASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) case GUEST_PENDING_DBG_EXCEPTIONS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) case GUEST_BNDCFGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) vmcs12->guest_pending_dbg_exceptions =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) if (kvm_mpx_supported())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) vmx->loaded_vmcs = &vmx->nested.vmcs02;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) vmx->loaded_vmcs = &vmx->vmcs01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) * Update the guest state fields of vmcs12 to reflect changes that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) * occurred while L2 was running. (The "IA-32e mode guest" bit of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) * VM-entry controls is also updated, since this is really a guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) * state bit.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) if (vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) vmcs12->guest_rsp = kvm_rsp_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) vmcs12->guest_rip = kvm_rip_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) vmcs12->guest_interruptibility_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) if (nested_cpu_has_preemption_timer(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) !vmx->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) vmcs12->vmx_preemption_timer_value =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) vmx_get_preemption_timer_value(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) * In some cases (usually, nested EPT), L2 is allowed to change its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) * own CR3 without exiting. If it has changed it, we must keep it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) * Additionally, restore L2's PDPTR to vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) if (nested_cpu_has_vid(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) vmcs12->vm_entry_controls =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) vmcs12->guest_ia32_efer = vcpu->arch.efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) * and this function updates it to reflect the changes to the guest state while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) * L2 was running (and perhaps made some exits which were handled directly by L0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) * without going back to L1), and to reflect the exit reason.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) * Note that we do not have to copy here all VMCS fields, just those that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) * could have changed by the L2 guest or the exit - i.e., the guest-state and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) * exit-information fields only. Other fields are modified by L1 with VMWRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) * which already writes to vmcs12 directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) u32 vm_exit_reason, u32 exit_intr_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) unsigned long exit_qualification)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) /* update exit information fields: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) vmcs12->vm_exit_reason = vm_exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) vmcs12->exit_qualification = exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) vmcs12->vm_exit_intr_info = exit_intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) vmcs12->idt_vectoring_info_field = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) vmcs12->launch_state = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) /* vm_entry_intr_info_field is cleared on exit. Emulate this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) * instead of reading the real value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) * Transfer the event that L0 or L1 may wanted to inject into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) * L2 to IDT_VECTORING_INFO_FIELD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) vmcs12_save_pending_event(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) * According to spec, there's no need to store the guest's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) * MSRs if the exit is due to a VM-entry failure that occurs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) * during or after loading the guest state. Since this exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) * does not fall in that category, we need to save the MSRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) if (nested_vmx_store_msr(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) vmcs12->vm_exit_msr_store_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) vmcs12->vm_exit_msr_store_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) nested_vmx_abort(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) VMX_ABORT_SAVE_GUEST_MSR_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) * Drop what we picked up for L2 via vmx_complete_interrupts. It is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) * preserved above and would only end up incorrectly in L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) vcpu->arch.nmi_injected = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) kvm_clear_exception_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) kvm_clear_interrupt_queue(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) * A part of what we need to when the nested L2 guest exits and we want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) * run its L1 parent, is to reset L1's guest state to the host state specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) * in vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) * This function is to be called not only on normal nested exit, but also on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) * Failures During or After Loading Guest State").
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) * This function should be called when the active VMCS is L1's (vmcs01).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) enum vm_entry_failure_code ignored;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) struct kvm_segment seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) vcpu->arch.efer = vmcs12->host_ia32_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) vcpu->arch.efer |= (EFER_LMA | EFER_LME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) vmx_set_efer(vcpu, vcpu->arch.efer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) kvm_rsp_write(vcpu, vmcs12->host_rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) kvm_rip_write(vcpu, vmcs12->host_rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) vmx_set_interrupt_shadow(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) * actually changed, because vmx_set_cr0 refers to efer set above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) * CR0_GUEST_HOST_MASK is already set in the original vmcs01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) * (KVM doesn't change it);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) vmx_set_cr0(vcpu, vmcs12->host_cr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) /* Same as above - no reason to call set_cr4_guest_host_mask(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) vmx_set_cr4(vcpu, vmcs12->host_cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) nested_ept_uninit_mmu_context(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) * Only PDPTE load can fail as the value of cr3 was checked on entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) * couldn't have changed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &ignored))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) nested_vmx_transition_tlb_flush(vcpu, vmcs12, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) vmcs_write64(GUEST_BNDCFGS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) vcpu->arch.pat = vmcs12->host_ia32_pat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) vmcs12->host_ia32_perf_global_ctrl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) /* Set L1 segment info according to Intel SDM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) 27.5.2 Loading Host Segment and Descriptor-Table Registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) seg = (struct kvm_segment) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) .base = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) .limit = 0xFFFFFFFF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) .selector = vmcs12->host_cs_selector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) .type = 11,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) .present = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) .s = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) .g = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) seg.l = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) seg.db = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) seg = (struct kvm_segment) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) .base = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) .limit = 0xFFFFFFFF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) .type = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) .present = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) .s = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) .db = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) .g = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) seg.selector = vmcs12->host_ds_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) seg.selector = vmcs12->host_es_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) seg.selector = vmcs12->host_ss_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) seg.selector = vmcs12->host_fs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) seg.base = vmcs12->host_fs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) seg.selector = vmcs12->host_gs_selector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) seg.base = vmcs12->host_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) seg = (struct kvm_segment) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) .base = vmcs12->host_tr_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) .limit = 0x67,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) .selector = vmcs12->host_tr_selector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) .type = 11,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) .present = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) kvm_set_dr(vcpu, 7, 0x400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) vmx_update_msr_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) vmcs12->vm_exit_msr_load_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) struct vmx_uret_msr *efer_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) return vmcs_read64(GUEST_IA32_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) if (cpu_has_load_ia32_efer())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) return host_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) return vmx->msr_autoload.guest.val[i].value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) efer_msr = vmx_find_uret_msr(vmx, MSR_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) if (efer_msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) return efer_msr->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) return host_efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) struct vmx_msr_entry g, h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) gpa_t gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) u32 i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) * as vmcs01.GUEST_DR7 contains a userspace defined value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) * and vcpu->arch.dr7 is not squirreled away before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) * nested VMENTER (not worth adding a variable in nested_vmx).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) kvm_set_dr(vcpu, 7, DR7_FIXED_1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) * Note that calling vmx_set_{efer,cr0,cr4} is important as they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) * handle a variety of side effects to KVM's software model.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) vcpu->arch.cr0_guest_owned_bits = KVM_POSSIBLE_CR0_GUEST_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) nested_ept_uninit_mmu_context(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) * from vmcs01 (if necessary). The PDPTRs are not loaded on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) * VMFail, like everything else we just need to ensure our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) * software model is up-to-date.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) if (enable_ept && is_pae_paging(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) ept_save_pdptrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) kvm_mmu_reset_context(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) if (cpu_has_vmx_msr_bitmap())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) vmx_update_msr_bitmap(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) * This nasty bit of open coding is a compromise between blindly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) * loading L1's MSRs using the exit load lists (incorrect emulation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) * of VMFail), leaving the nested VM's MSRs in the software model
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) * (incorrect behavior) and snapshotting the modified MSRs (too
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) * expensive since the lists are unbound by hardware). For each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) * MSR that was (prematurely) loaded from the nested VMEntry load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) * list, reload it from the exit load list if it exists and differs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) * from the guest value. The intent is to stuff host state as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) * silently as possible, not to fully process the exit load list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) "%s read MSR index failed (%u, 0x%08llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) __func__, i, gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) goto vmabort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) "%s read MSR failed (%u, 0x%08llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) __func__, j, gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) goto vmabort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) if (h.index != g.index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) if (h.value == g.value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) if (nested_vmx_load_msr_check(vcpu, &h)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) "%s check failed (%u, 0x%x, 0x%x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) __func__, j, h.index, h.reserved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) goto vmabort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) if (kvm_set_msr(vcpu, h.index, h.value)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) "%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) __func__, j, h.index, h.value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) goto vmabort;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) vmabort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) * and modify vmcs12 to make it see what it would expect to see there if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) u32 exit_intr_info, unsigned long exit_qualification)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) /* trying to cancel vmlaunch/vmresume is a bug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) WARN_ON_ONCE(vmx->nested.nested_run_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) * Enlightened VMCS after migration and we still need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) * do that when something is forcing L2->L1 exit prior to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) * the first L2 run.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) (void)nested_get_evmcs_page(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) /* Service the TLB flush request for L2 before switching to L1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) kvm_vcpu_flush_tlb_current(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) * up-to-date before switching to L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) if (enable_ept && is_pae_paging(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) vmx_ept_load_pdptrs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) leave_guest_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) if (nested_cpu_has_preemption_timer(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) if (likely(!vmx->fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) sync_vmcs02_to_vmcs12(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) if (vm_exit_reason != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) prepare_vmcs12(vcpu, vmcs12, vm_exit_reason,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) exit_intr_info, exit_qualification);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) * Must happen outside of sync_vmcs02_to_vmcs12() as it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) * also be used to capture vmcs12 cache as part of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) * capturing nVMX state for snapshot (migration).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) * Otherwise, this flush will dirty guest memory at a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) * point it is already assumed by user-space to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) * immutable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) * The only expected VM-instruction error is "VM entry with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) * invalid control field(s)." Anything else indicates a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) * problem with L0. And we should never get here with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) * VMFail of any type if early consistency checks are enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) VMXERR_ENTRY_INVALID_CONTROL_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) WARN_ON_ONCE(nested_early_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) vmx_switch_vmcs(vcpu, &vmx->vmcs01);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) /* Update any VMCS fields that might have changed while L2 ran */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) if (vmx->nested.l1_tpr_threshold != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) if (kvm_has_tsc_control)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) decache_tsc_multiplier(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) if (vmx->nested.change_vmcs01_virtual_apic_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) vmx->nested.change_vmcs01_virtual_apic_mode = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) vmx_set_virtual_apic_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) /* Unpin physical memory we referred to in vmcs02 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) if (vmx->nested.apic_access_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) kvm_release_page_clean(vmx->nested.apic_access_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) vmx->nested.apic_access_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) vmx->nested.pi_desc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) if (vmx->nested.reload_vmcs01_apic_access_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) vmx->nested.reload_vmcs01_apic_access_page = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) if ((vm_exit_reason != -1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) (enable_shadow_vmcs || vmx->nested.hv_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) vmx->nested.need_vmcs12_to_shadow_sync = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) /* in case we halted in L2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) if (likely(!vmx->fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) nested_exit_intr_ack_set(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) int irq = kvm_cpu_get_interrupt(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) WARN_ON(irq < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) vmcs12->vm_exit_intr_info = irq |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) if (vm_exit_reason != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) vmcs12->exit_qualification,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) vmcs12->idt_vectoring_info_field,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) vmcs12->vm_exit_intr_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) vmcs12->vm_exit_intr_error_code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) KVM_ISA_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) load_vmcs12_host_state(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) * After an early L2 VM-entry failure, we're now back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) * in L1 which thinks it just finished a VMLAUNCH or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) * VMRESUME instruction, so we need to set the failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) * flag and the VM-instruction error field of the VMCS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) * accordingly, and skip the emulated instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) * Restore L1's host state to KVM's software model. We're here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) * because a consistency check was caught by hardware, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) * means some amount of guest state has been propagated to KVM's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) * model and needs to be unwound to the host's state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) nested_vmx_restore_host_state(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) vmx->fail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) * Decode the memory-address operand of a vmx instruction, as recorded on an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) * exit caused by such an instruction (run by a guest hypervisor).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) * On success, returns 0. When the operand is invalid, returns 1 and throws
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) * #UD, #GP, or #SS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) u32 vmx_instruction_info, bool wr, int len, gva_t *ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) gva_t off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) bool exn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) struct kvm_segment s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) * According to Vol. 3B, "Information for VM Exits Due to Instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) * Execution", on an exit, vmx_instruction_info holds most of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) * addressing components of the operand. Only the displacement part
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) * For how an actual address is calculated from all these components,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) * refer to Vol. 1, "Operand Addressing".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) int scaling = vmx_instruction_info & 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) int addr_size = (vmx_instruction_info >> 7) & 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) bool is_reg = vmx_instruction_info & (1u << 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) int seg_reg = (vmx_instruction_info >> 15) & 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) int index_reg = (vmx_instruction_info >> 18) & 0xf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) bool index_is_valid = !(vmx_instruction_info & (1u << 22));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) int base_reg = (vmx_instruction_info >> 23) & 0xf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) bool base_is_valid = !(vmx_instruction_info & (1u << 27));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) if (is_reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) /* Addr = segment_base + offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) /* offset = base + [index * scale] + displacement */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) off = exit_qualification; /* holds the displacement */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) if (addr_size == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) off = (gva_t)sign_extend64(off, 31);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) else if (addr_size == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) off = (gva_t)sign_extend64(off, 15);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) if (base_is_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) off += kvm_register_readl(vcpu, base_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) if (index_is_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) off += kvm_register_readl(vcpu, index_reg) << scaling;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) vmx_get_segment(vcpu, &s, seg_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) * The effective address, i.e. @off, of a memory operand is truncated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) * based on the address size of the instruction. Note that this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) * the *effective address*, i.e. the address prior to accounting for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) * the segment's base.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) if (addr_size == 1) /* 32 bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) off &= 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) else if (addr_size == 0) /* 16 bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) off &= 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) /* Checks for #GP/#SS exceptions. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) exn = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) if (is_long_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) * The virtual/linear address is never truncated in 64-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) * mode, e.g. a 32-bit address size can yield a 64-bit virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) * address when using FS/GS with a non-zero base.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) *ret = s.base + off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) *ret = off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) /* Long mode: #GP(0)/#SS(0) if the memory address is in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) * non-canonical form. This is the only check on the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) * destination for long mode!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) exn = is_noncanonical_address(*ret, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) * When not in long mode, the virtual/linear address is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) * unconditionally truncated to 32 bits regardless of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) * address size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) *ret = (s.base + off) & 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) /* Protected mode: apply checks for segment validity in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) * following order:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) * - segment type check (#GP(0) may be thrown)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) * - usability check (#GP(0)/#SS(0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) * - limit check (#GP(0)/#SS(0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) if (wr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) /* #GP(0) if the destination operand is located in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) * read-only data segment or any code segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699) exn = ((s.type & 0xa) == 0 || (s.type & 8));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) /* #GP(0) if the source operand is located in an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) * execute-only code segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) exn = ((s.type & 0xa) == 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) if (exn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) exn = (s.unusable != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) * Protected mode: #GP(0)/#SS(0) if the memory operand is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) * outside the segment limit. All CPUs that support VMX ignore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) * limit checks for flat segments, i.e. segments with base==0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) * limit==0xffffffff and of type expand-up data or code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) if (!(s.base == 0 && s.limit == 0xffffffff &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) ((s.type & 8) || !(s.type & 4))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) exn = exn || ((u64)off + len - 1 > s.limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) if (exn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) kvm_queue_exception_e(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) seg_reg == VCPU_SREG_SS ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) SS_VECTOR : GP_VECTOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) struct vcpu_vmx *vmx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) if (!nested_vmx_allowed(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) if (kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) vmx->nested.msrs.entry_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) vmx->nested.msrs.exit_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) vmx->nested.msrs.entry_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) vmx->nested.msrs.exit_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) int *ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) vmcs_read32(VMX_INSTRUCTION_INFO), false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) sizeof(*vmpointer), &gva)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) *ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) if (r != X86EMUL_CONTINUE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) *ret = kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) * Allocate a shadow VMCS and associate it with the currently loaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) * VMCS, unless such a shadow VMCS already exists. The newly allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) * VMCS is also VMCLEARed, so that it is ready for use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) * We should allocate a shadow vmcs for vmcs01 only when L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) * executes VMXON and free it when L1 executes VMXOFF.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) * As it is invalid to execute VMXON twice, we shouldn't reach
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) * here when vmcs01 already have an allocated shadow vmcs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) if (!loaded_vmcs->shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) if (loaded_vmcs->shadow_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) vmcs_clear(loaded_vmcs->shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) return loaded_vmcs->shadow_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) static int enter_vmx_operation(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) goto out_vmcs02;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) if (!vmx->nested.cached_vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) goto out_cached_vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) if (!vmx->nested.cached_shadow_vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) goto out_cached_shadow_vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) goto out_shadow_vmcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824) hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) HRTIMER_MODE_ABS_PINNED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) vmx->nested.vpid02 = allocate_vpid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) vmx->nested.vmcs02_initialized = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) vmx->nested.vmxon = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) if (vmx_pt_mode_is_host_guest()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) vmx->pt_desc.guest.ctl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) pt_update_intercept_for_msr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) out_shadow_vmcs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841) kfree(vmx->nested.cached_shadow_vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) out_cached_shadow_vmcs12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) kfree(vmx->nested.cached_vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) out_cached_vmcs12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) free_loaded_vmcs(&vmx->nested.vmcs02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) out_vmcs02:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) * Emulate the VMXON instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) * Currently, we just remember that VMX is active, and do not save or even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) * inspect the argument to VMXON (the so-called "VMXON pointer") because we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) * do not currently need to store anything in that guest-allocated memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) * argument is different from the VMXON pointer (which the spec says they do).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) static int handle_vmon(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) gpa_t vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) uint32_t revision;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) * The Intel VMX Instruction Reference lists a bunch of bits that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) * prerequisite to running VMXON, most notably cr4.VMXE must be set to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) * 1 (see vmx_set_cr4() for when we allow the guest to set this).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) * Otherwise, we should fail with #UD. But most faulting conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) * have already been checked by hardware, prior to the VM-exit for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) * VMXON. We do test guest cr4.VMXE because processor CR4 always has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) * that bit set to 1 in non-root mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) /* CPL=0 must be checked manually. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) if (vmx_get_cpl(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) kvm_inject_gp(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) if (vmx->nested.vmxon)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) != VMXON_NEEDED_FEATURES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) kvm_inject_gp(vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) * SDM 3: 24.11.5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) * The first 4 bytes of VMXON region contain the supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) * VMCS revision identifier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) * which replaces physical address width with 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) if (!page_address_valid(vcpu, vmptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) revision != VMCS12_REVISION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) vmx->nested.vmxon_ptr = vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) ret = enter_vmx_operation(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) if (vmx->nested.current_vmptr == -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) /* copy to memory all shadowed fields in case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) they were modified */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) copy_shadow_to_vmcs12(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) vmx_disable_shadow_vmcs(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) vmx->nested.posted_intr_nv = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) /* Flush VMCS12 to guest memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) kvm_vcpu_write_guest_page(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) vmx->nested.current_vmptr >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) vmx->nested.current_vmptr = -1ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) /* Emulate the VMXOFF instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) static int handle_vmoff(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) free_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) /* Process a latched INIT during time CPU was in VMX operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) kvm_make_request(KVM_REQ_EVENT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) /* Emulate the VMCLEAR instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) static int handle_vmclear(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) u32 zero = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) gpa_t vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) u64 evmcs_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) if (!page_address_valid(vcpu, vmptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) if (vmptr == vmx->nested.vmxon_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) * When Enlightened VMEntry is enabled on the calling CPU we treat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) * memory area pointer by vmptr as Enlightened VMCS (as there's no good
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) * way to distinguish it from VMCS12) and we must not corrupt it by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) * writing to the non-existent 'launch_state' field. The area doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) * have to be the currently active EVMCS on the calling CPU and there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) * nothing KVM has to do to transition it from 'active' to 'non-active'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) * state. It is possible that the area will stay mapped as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) * vmx->nested.hv_evmcs but this shouldn't be a problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) if (likely(!vmx->nested.enlightened_vmcs_enabled ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) if (vmptr == vmx->nested.current_vmptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) nested_release_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) kvm_vcpu_write_guest(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) vmptr + offsetof(struct vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) launch_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) &zero, sizeof(zero));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) /* Emulate the VMLAUNCH instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) static int handle_vmlaunch(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) return nested_vmx_run(vcpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) /* Emulate the VMRESUME instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) static int handle_vmresume(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) return nested_vmx_run(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) static int handle_vmread(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) : get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) unsigned long field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) u64 value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) gva_t gva = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) short offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) int len, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042) * In VMX non-root operation, when the VMCS-link pointer is -1ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) * any VMREAD sets the ALU flags for VMfailInvalid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) if (vmx->nested.current_vmptr == -1ull ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) /* Decode instruction info and find the field to read */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) offset = vmcs_field_to_offset(field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) if (offset < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) /* Read the field, zero-extended to a u64 value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) value = vmcs12_read_any(vmcs12, field, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064) * Now copy part of this value to register or memory, as requested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) * Note that the number of bits actually copied is 32 or 64 depending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) * on the guest's mode (32 or 64 bit), not on the given field's length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) if (instr_info & BIT(10)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) kvm_register_writel(vcpu, (((instr_info) >> 3) & 0xf), value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) len = is_64_bit_mode(vcpu) ? 8 : 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) if (get_vmx_mem_address(vcpu, exit_qualification,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) instr_info, true, len, &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) /* _system ok, nested_vmx_check_permission has verified cpl=0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) if (r != X86EMUL_CONTINUE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) return kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) static bool is_shadow_field_rw(unsigned long field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) switch (field) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) #define SHADOW_FIELD_RW(x, y) case x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) #include "vmcs_shadow_fields.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) static bool is_shadow_field_ro(unsigned long field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) switch (field) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) #define SHADOW_FIELD_RO(x, y) case x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) #include "vmcs_shadow_fields.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) static int handle_vmwrite(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110) struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) : get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) unsigned long field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) short offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) int len, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) * The value to write might be 32 or 64 bits, depending on L1's long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) * mode, and eventually we need to write that into a field of several
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) * possible lengths. The code below first zero-extends the value to 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) * bit (value), and then copies only the appropriate number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) * bits into the vmcs12 field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) u64 value = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) * In VMX non-root operation, when the VMCS-link pointer is -1ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) * any VMWRITE sets the ALU flags for VMfailInvalid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) if (vmx->nested.current_vmptr == -1ull ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) return nested_vmx_failInvalid(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) if (instr_info & BIT(10))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) value = kvm_register_readl(vcpu, (((instr_info) >> 3) & 0xf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) len = is_64_bit_mode(vcpu) ? 8 : 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) if (get_vmx_mem_address(vcpu, exit_qualification,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147) instr_info, false, len, &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) r = kvm_read_guest_virt(vcpu, gva, &value, len, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150) if (r != X86EMUL_CONTINUE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) return kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) offset = vmcs_field_to_offset(field);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) if (offset < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158) return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) * If the vCPU supports "VMWRITE to any supported field in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) * VMCS," then the "read-only" fields are actually read/write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) if (vmcs_field_readonly(field) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) !nested_cpu_has_vmwrite_any_field(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) * Ensure vmcs12 is up-to-date before any VMWRITE that dirties
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) * vmcs12, else we may crush a field or consume a stale value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) * Some Intel CPUs intentionally drop the reserved bits of the AR byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) * fields on VMWRITE. Emulate this behavior to ensure consistent KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) * behavior regardless of the underlying hardware, e.g. if an AR_BYTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) * from L1 will return a different value than VMREAD from L2 (L1 sees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) * the stripped down value, L2 sees the full value as stored by KVM).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) value &= 0x1f0ff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) vmcs12_write_any(vmcs12, field, offset, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) * Do not track vmcs12 dirty-state if in guest-mode as we actually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) * by L1 without a vmexit are always updated in the vmcs02, i.e. don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) * L1 can read these fields without exiting, ensure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) * shadow VMCS is up-to-date.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) if (enable_shadow_vmcs && is_shadow_field_ro(field)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) vmcs_load(vmx->vmcs01.shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) __vmcs_writel(field, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) vmcs_clear(vmx->vmcs01.shadow_vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206) vmcs_load(vmx->loaded_vmcs->vmcs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) vmx->nested.dirty_vmcs12 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) vmx->nested.current_vmptr = vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) vmcs_write64(VMCS_LINK_POINTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) __pa(vmx->vmcs01.shadow_vmcs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) vmx->nested.need_vmcs12_to_shadow_sync = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) vmx->nested.dirty_vmcs12 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) /* Emulate the VMPTRLD instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) static int handle_vmptrld(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) gpa_t vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237) if (nested_vmx_get_vmptr(vcpu, &vmptr, &r))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240) if (!page_address_valid(vcpu, vmptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) if (vmptr == vmx->nested.vmxon_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) /* Forbid normal VMPTRLD if Enlightened version was used */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) if (vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) if (vmx->nested.current_vmptr != vmptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251) struct kvm_host_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) struct vmcs12 *new_vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) * Reads from an unbacked page return all 1s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) * which means that the 32 bits located at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) * given physical address won't match the required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) * VMCS12_REVISION identifier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) new_vmcs12 = map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) if (new_vmcs12->hdr.revision_id != VMCS12_REVISION ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) (new_vmcs12->hdr.shadow_vmcs &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269) !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) kvm_vcpu_unmap(vcpu, &map, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) nested_release_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) * Load VMCS12 from guest memory since it is not already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) * cached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) kvm_vcpu_unmap(vcpu, &map, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) set_current_vmptr(vmx, vmptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) /* Emulate the VMPTRST instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) static int handle_vmptrst(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) unsigned long exit_qual = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294) u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) if (unlikely(to_vmx(vcpu)->nested.hv_evmcs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) true, sizeof(gpa_t), &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) sizeof(gpa_t), &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) if (r != X86EMUL_CONTINUE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) return kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) #define EPTP_PA_MASK GENMASK_ULL(51, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) return VALID_PAGE(root_hpa) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) /* Emulate the INVEPT instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) static int handle_invept(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) u32 vmx_instruction_info, types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) unsigned long type, roots_to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) struct kvm_mmu *mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) u64 eptp, gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) } operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) int i, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) if (!(vmx->nested.msrs.secondary_ctls_high &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) SECONDARY_EXEC_ENABLE_EPT) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355) if (type >= 32 || !(types & (1 << type)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) /* According to the Intel VMX instruction reference, the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) * operand is read even if it isn't needed (e.g., for type==global)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) vmx_instruction_info, false, sizeof(operand), &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) if (r != X86EMUL_CONTINUE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) return kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) * Nested EPT roots are always held through guest_mmu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) * not root_mmu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) mmu = &vcpu->arch.guest_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) case VMX_EPT_EXTENT_CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376) if (!nested_vmx_check_eptp(vcpu, operand.eptp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) roots_to_free = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) if (nested_ept_root_matches(mmu->root_hpa, mmu->root_pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) operand.eptp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) roots_to_free |= KVM_MMU_ROOT_CURRENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) if (nested_ept_root_matches(mmu->prev_roots[i].hpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387) mmu->prev_roots[i].pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) operand.eptp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389) roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392) case VMX_EPT_EXTENT_GLOBAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) roots_to_free = KVM_MMU_ROOTS_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400) if (roots_to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) kvm_mmu_free_roots(vcpu, mmu, roots_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) static int handle_invvpid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) u32 vmx_instruction_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) unsigned long type, types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) gva_t gva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) struct x86_exception e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) u64 vpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) u64 gla;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) } operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417) u16 vpid02;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) if (!(vmx->nested.msrs.secondary_ctls_high &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) SECONDARY_EXEC_ENABLE_VPID) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) if (!nested_vmx_check_permission(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430) vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433) types = (vmx->nested.msrs.vpid_caps &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436) if (type >= 32 || !(types & (1 << type)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) /* according to the intel vmx instruction reference, the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) * operand is read even if it isn't needed (e.g., for type==global)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) vmx_instruction_info, false, sizeof(operand), &gva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446) r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) if (r != X86EMUL_CONTINUE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448) return kvm_handle_memory_failure(vcpu, r, &e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450) if (operand.vpid >> 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) vpid02 = nested_get_vpid02(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457) if (!operand.vpid ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) is_noncanonical_address(operand.gla, vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461) vpid_sync_vcpu_addr(vpid02, operand.gla);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) case VMX_VPID_EXTENT_SINGLE_CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465) if (!operand.vpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466) return nested_vmx_fail(vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) vpid_sync_context(vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470) case VMX_VPID_EXTENT_ALL_CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) vpid_sync_context(vpid02);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479) * Sync the shadow page tables if EPT is disabled, L1 is invalidating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) * linear mappings for L2 (tagged with L2's VPID). Free all roots as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481) * VPIDs are not tracked in the MMU role.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483) * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) * an MMU when EPT is disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486) * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) if (!enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489) kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490) KVM_MMU_ROOTS_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) return nested_vmx_succeed(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495) static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498) u32 index = kvm_rcx_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) u64 new_eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501) if (!nested_cpu_has_eptp_switching(vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) !nested_cpu_has_ept(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) if (index >= VMFUNC_EPTP_ENTRIES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509) &new_eptp, index * 8, 8))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) * If the (L2) guest does a vmfunc to the currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514) * active ept pointer, we don't have to do anything else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516) if (vmcs12->ept_pointer != new_eptp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) if (!nested_vmx_check_eptp(vcpu, new_eptp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) vmcs12->ept_pointer = new_eptp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528) static int handle_vmfunc(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531) struct vmcs12 *vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) u32 function = kvm_rax_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) * VMFUNC is only supported for nested guests, but we always enable the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536) * secondary control for simplicity; for non-nested mode, fake that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) * didn't by injecting #UD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539) if (!is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540) kvm_queue_exception(vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544) vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545) if (!(vmcs12->vm_function_control & BIT_ULL(function)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548) switch (function) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550) if (nested_vmx_eptp_switching(vcpu, vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556) return kvm_skip_emulated_instruction(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) * This is effectively a reflected VM-Exit, as opposed to a synthesized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) * EXIT_REASON_VMFUNC as the exit reason.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) nested_vmx_vmexit(vcpu, vmx->exit_reason.full,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565) vmx_get_intr_info(vcpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566) vmx_get_exit_qual(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571) * Return true if an IO instruction with the specified port and size should cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) * a VM-exit into L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574) bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575) int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578) gpa_t bitmap, last_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579) u8 b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) last_bitmap = (gpa_t)-1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) b = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584) while (size > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) if (port < 0x8000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) bitmap = vmcs12->io_bitmap_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587) else if (port < 0x10000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588) bitmap = vmcs12->io_bitmap_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) bitmap += (port & 0x7fff) / 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593) if (last_bitmap != bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594) if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596) if (b & (1 << (port & 7)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) port++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) size--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) last_bitmap = bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) unsigned long exit_qualification;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611) unsigned short port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614) if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619) port = exit_qualification >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) size = (exit_qualification & 7) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622) return nested_vmx_check_io_bitmaps(vcpu, port, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) * Return 1 if we should exit from L2 to L1 to handle an MSR access,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) * rather than handle it ourselves in L0. I.e., check whether L1 expressed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628) * disinterest in the current event (read or write a specific MSR) by using an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629) * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631) static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) struct vmcs12 *vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) union vmx_exit_reason exit_reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) u32 msr_index = kvm_rcx_read(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) gpa_t bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643) * for the four combinations of read/write and low/high MSR numbers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644) * First we need to figure out which of the four to use:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) bitmap = vmcs12->msr_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647) if (exit_reason.basic == EXIT_REASON_MSR_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) bitmap += 2048;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) if (msr_index >= 0xc0000000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650) msr_index -= 0xc0000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) bitmap += 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654) /* Then read the msr_index'th bit from this bitmap: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655) if (msr_index < 1024*8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) unsigned char b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657) if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659) return 1 & (b >> (msr_index & 7));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661) return true; /* let L1 handle the wrong parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665) * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666) * rather than handle it ourselves in L0. I.e., check if L1 wanted to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667) * intercept (via guest_host_mask etc.) the current event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669) static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670) struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672) unsigned long exit_qualification = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) int cr = exit_qualification & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) int reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677) switch ((exit_qualification >> 4) & 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678) case 0: /* mov to cr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) reg = (exit_qualification >> 8) & 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680) val = kvm_register_readl(vcpu, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681) switch (cr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683) if (vmcs12->cr0_guest_host_mask &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) (val ^ vmcs12->cr0_read_shadow))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692) if (vmcs12->cr4_guest_host_mask &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) (vmcs12->cr4_read_shadow ^ val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697) if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) case 2: /* clts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703) if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704) (vmcs12->cr0_read_shadow & X86_CR0_TS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) case 1: /* mov from cr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708) switch (cr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) if (vmcs12->cpu_based_vm_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) CPU_BASED_CR3_STORE_EXITING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715) if (vmcs12->cpu_based_vm_exec_control &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716) CPU_BASED_CR8_STORE_EXITING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721) case 3: /* lmsw */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) * lmsw can change bits 1..3 of cr0, and only set bit 0 of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724) * cr0. Other attempted changes are ignored, with no exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727) if (vmcs12->cr0_guest_host_mask & 0xe &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728) (val ^ vmcs12->cr0_read_shadow))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) if ((vmcs12->cr0_guest_host_mask & 0x1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731) !(vmcs12->cr0_read_shadow & 0x1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) (val & 0x1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) struct vmcs12 *vmcs12, gpa_t bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742) u32 vmx_instruction_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) unsigned long field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744) u8 b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) if (!nested_cpu_has_shadow_vmcs(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749) /* Decode instruction info and find the field to access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750) vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751) field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) /* Out-of-range fields always cause a VM exit from L2 to L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754) if (field >> 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) return 1 & (b >> (field & 7));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765) u32 entry_intr_info = vmcs12->vm_entry_intr_info_field;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) if (nested_cpu_has_mtf(vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771) * An MTF VM-exit may be injected into the guest by setting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772) * interruption-type to 7 (other event) and the vector field to 0. Such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) * is the case regardless of the 'monitor trap flag' VM-execution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) * control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776) return entry_intr_info == (INTR_INFO_VALID_MASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777) | INTR_TYPE_OTHER_EVENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) * Return true if L0 wants to handle an exit from L2 regardless of whether or not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782) * L1 wants the exit. Only call this when in is_guest_mode (L2).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) union vmx_exit_reason exit_reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787) u32 intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789) switch ((u16)exit_reason.basic) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) case EXIT_REASON_EXCEPTION_NMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791) intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) if (is_nmi(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794) else if (is_page_fault(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795) return vcpu->arch.apf.host_apf_flags ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) vmx_need_pf_intercept(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) else if (is_debug(intr_info) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798) vcpu->guest_debug &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801) else if (is_breakpoint(intr_info) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802) vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804) else if (is_alignment_check(intr_info) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) !vmx_guest_inject_ac(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808) case EXIT_REASON_EXTERNAL_INTERRUPT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) case EXIT_REASON_MCE_DURING_VMENTRY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812) case EXIT_REASON_EPT_VIOLATION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814) * L0 always deals with the EPT violation. If nested EPT is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815) * used, and the nested mmu code discovers that the address is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) * missing in the guest EPT table (EPT12), the EPT violation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) * will be injected with nested_ept_inject_page_fault()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820) case EXIT_REASON_EPT_MISCONFIG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) * L2 never uses directly L1's EPT, but rather L0's own EPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823) * table (shadow on EPT) or a merged EPT table that L0 built
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) * (EPT on EPT). So any problems with the structure of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825) * table is L0's fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) case EXIT_REASON_PREEMPTION_TIMER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830) case EXIT_REASON_PML_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) /* We emulate PML support to L1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) case EXIT_REASON_VMFUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) /* VM functions are emulated through L2->L0 vmexits. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836) case EXIT_REASON_ENCLS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) /* SGX is never exposed to L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) * Return 1 if L1 wants to intercept an exit from L2. Only call this when in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847) * is_guest_mode (L2).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) union vmx_exit_reason exit_reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853) u32 intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) switch ((u16)exit_reason.basic) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856) case EXIT_REASON_EXCEPTION_NMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857) intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858) if (is_nmi(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860) else if (is_page_fault(intr_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) return vmcs12->exception_bitmap &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) (1u << (intr_info & INTR_INFO_VECTOR_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864) case EXIT_REASON_EXTERNAL_INTERRUPT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) return nested_exit_on_intr(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866) case EXIT_REASON_TRIPLE_FAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) case EXIT_REASON_INTERRUPT_WINDOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870) case EXIT_REASON_NMI_WINDOW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871) return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) case EXIT_REASON_TASK_SWITCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874) case EXIT_REASON_CPUID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876) case EXIT_REASON_HLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877) return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) case EXIT_REASON_INVD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880) case EXIT_REASON_INVLPG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882) case EXIT_REASON_RDPMC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883) return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) case EXIT_REASON_RDRAND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886) case EXIT_REASON_RDSEED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888) case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) case EXIT_REASON_VMREAD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891) return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892) vmcs12->vmread_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) case EXIT_REASON_VMWRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894) return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895) vmcs12->vmwrite_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898) case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899) case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900) case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) * VMX instructions trap unconditionally. This allows L1 to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903) * emulate them for its L2 guest, i.e., allows 3-level nesting!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906) case EXIT_REASON_CR_ACCESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) return nested_vmx_exit_handled_cr(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) case EXIT_REASON_DR_ACCESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909) return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) case EXIT_REASON_IO_INSTRUCTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) return nested_vmx_exit_handled_io(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912) case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913) return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) case EXIT_REASON_MSR_READ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915) case EXIT_REASON_MSR_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917) case EXIT_REASON_INVALID_STATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919) case EXIT_REASON_MWAIT_INSTRUCTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921) case EXIT_REASON_MONITOR_TRAP_FLAG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922) return nested_vmx_exit_handled_mtf(vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) case EXIT_REASON_MONITOR_INSTRUCTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924) return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925) case EXIT_REASON_PAUSE_INSTRUCTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) nested_cpu_has2(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) SECONDARY_EXEC_PAUSE_LOOP_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) case EXIT_REASON_MCE_DURING_VMENTRY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) case EXIT_REASON_TPR_BELOW_THRESHOLD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932) return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) case EXIT_REASON_APIC_ACCESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934) case EXIT_REASON_APIC_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) case EXIT_REASON_EOI_INDUCED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) * The controls for "virtualize APIC accesses," "APIC-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938) * register virtualization," and "virtual-interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939) * delivery" only come from vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5941) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5942) case EXIT_REASON_INVPCID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5943) return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5944) nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5945) nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5946) case EXIT_REASON_WBINVD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5947) return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5948) case EXIT_REASON_XSETBV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5949) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5950) case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5951) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5952) * This should never happen, since it is not possible to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5953) * set XSS to a non-zero value---neither in L1 nor in L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5954) * If if it were, XSS would have to be checked against
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5955) * the XSS exit bitmap in vmcs12.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5956) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5957) return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5958) case EXIT_REASON_UMWAIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5959) case EXIT_REASON_TPAUSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5960) return nested_cpu_has2(vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5961) SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5962) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5963) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5967) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5968) * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5969) * reflected into L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5970) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5971) bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5972) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5973) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5974) union vmx_exit_reason exit_reason = vmx->exit_reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5975) unsigned long exit_qual;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5976) u32 exit_intr_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5978) WARN_ON_ONCE(vmx->nested.nested_run_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5980) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5981) * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5982) * has already loaded L2's state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5983) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5984) if (unlikely(vmx->fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5985) trace_kvm_nested_vmenter_failed(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5986) "hardware VM-instruction error: ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5987) vmcs_read32(VM_INSTRUCTION_ERROR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5988) exit_intr_info = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5989) exit_qual = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5990) goto reflect_vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5993) trace_kvm_nested_vmexit(exit_reason.full, vcpu, KVM_ISA_VMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5995) /* If L0 (KVM) wants the exit, it trumps L1's desires. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5996) if (nested_vmx_l0_wants_exit(vcpu, exit_reason))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5997) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5999) /* If L1 doesn't want the exit, handle it in L0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6000) if (!nested_vmx_l1_wants_exit(vcpu, exit_reason))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6001) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6003) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6004) * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6005) * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6006) * need to be synthesized by querying the in-kernel LAPIC, but external
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6007) * interrupts are never reflected to L1 so it's a non-issue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6008) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6009) exit_intr_info = vmx_get_intr_info(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6010) if (is_exception_with_error_code(exit_intr_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6011) struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6013) vmcs12->vm_exit_intr_error_code =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6014) vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6016) exit_qual = vmx_get_exit_qual(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6018) reflect_vmexit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6019) nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6020) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6023) static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6024) struct kvm_nested_state __user *user_kvm_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6025) u32 user_data_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6027) struct vcpu_vmx *vmx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6028) struct vmcs12 *vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6029) struct kvm_nested_state kvm_state = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6030) .flags = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6031) .format = KVM_STATE_NESTED_FORMAT_VMX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6032) .size = sizeof(kvm_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6033) .hdr.vmx.flags = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6034) .hdr.vmx.vmxon_pa = -1ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6035) .hdr.vmx.vmcs12_pa = -1ull,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6036) .hdr.vmx.preemption_timer_deadline = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6037) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6038) struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6039) &user_kvm_nested_state->data.vmx[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6041) if (!vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6042) return kvm_state.size + sizeof(*user_vmx_nested_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6044) vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6045) vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6047) if (nested_vmx_allowed(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6048) (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6049) kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6050) kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6052) if (vmx_has_valid_vmcs12(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6053) kvm_state.size += sizeof(user_vmx_nested_state->vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6055) if (vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6056) kvm_state.flags |= KVM_STATE_NESTED_EVMCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6058) if (is_guest_mode(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6059) nested_cpu_has_shadow_vmcs(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6060) vmcs12->vmcs_link_pointer != -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6061) kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6064) if (vmx->nested.smm.vmxon)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6065) kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6067) if (vmx->nested.smm.guest_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6068) kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6070) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6071) kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6073) if (vmx->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6074) kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6076) if (vmx->nested.mtf_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6077) kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6079) if (nested_cpu_has_preemption_timer(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6080) vmx->nested.has_preemption_timer_deadline) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6081) kvm_state.hdr.vmx.flags |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6082) KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6083) kvm_state.hdr.vmx.preemption_timer_deadline =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6084) vmx->nested.preemption_timer_deadline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6089) if (user_data_size < kvm_state.size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6090) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6092) if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6093) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6095) if (!vmx_has_valid_vmcs12(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6096) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6098) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6099) * When running L2, the authoritative vmcs12 state is in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6100) * vmcs02. When running L1, the authoritative vmcs12 state is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6101) * in the shadow or enlightened vmcs linked to vmcs01, unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6102) * need_vmcs12_to_shadow_sync is set, in which case, the authoritative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6103) * vmcs12 state is in the vmcs12 already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6105) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6106) sync_vmcs02_to_vmcs12(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6107) sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6108) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6109) copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6110) if (!vmx->nested.need_vmcs12_to_shadow_sync) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6111) if (vmx->nested.hv_evmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6112) copy_enlightened_to_vmcs12(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6113) else if (enable_shadow_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6114) copy_shadow_to_vmcs12(vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6118) BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6119) BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6122) * Copy over the full allocated size of vmcs12 rather than just the size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6123) * of the struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6125) if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6126) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6128) if (nested_cpu_has_shadow_vmcs(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6129) vmcs12->vmcs_link_pointer != -1ull) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6130) if (copy_to_user(user_vmx_nested_state->shadow_vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6131) get_shadow_vmcs12(vcpu), VMCS12_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6132) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6134) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6135) return kvm_state.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6138) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6139) * Forcibly leave nested mode in order to be able to reset the VCPU later on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6141) void vmx_leave_nested(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6143) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6144) to_vmx(vcpu)->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6145) nested_vmx_vmexit(vcpu, -1, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6147) free_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6150) static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6151) struct kvm_nested_state __user *user_kvm_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6152) struct kvm_nested_state *kvm_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6154) struct vcpu_vmx *vmx = to_vmx(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6155) struct vmcs12 *vmcs12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6156) enum vm_entry_failure_code ignored;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6157) struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6158) &user_kvm_nested_state->data.vmx[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6159) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6161) if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6162) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6164) if (kvm_state->hdr.vmx.vmxon_pa == -1ull) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6165) if (kvm_state->hdr.vmx.smm.flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6166) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6168) if (kvm_state->hdr.vmx.vmcs12_pa != -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6169) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6172) * KVM_STATE_NESTED_EVMCS used to signal that KVM should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6173) * enable eVMCS capability on vCPU. However, since then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6174) * code was changed such that flag signals vmcs12 should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6175) * be copied into eVMCS in guest memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6176) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6177) * To preserve backwards compatability, allow user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6178) * to set this flag even when there is no VMXON region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6180) if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6181) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6182) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6183) if (!nested_vmx_allowed(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6184) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6186) if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6187) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6190) if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6191) (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6192) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6194) if (kvm_state->hdr.vmx.smm.flags &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6195) ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6196) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6198) if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6199) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6202) * SMM temporarily disables VMX, so we cannot be in guest mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6203) * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6204) * must be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6206) if (is_smm(vcpu) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6207) (kvm_state->flags &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6208) (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6209) : kvm_state->hdr.vmx.smm.flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6210) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6212) if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6213) !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6214) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6216) if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6217) (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6218) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6220) vmx_leave_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6222) if (kvm_state->hdr.vmx.vmxon_pa == -1ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6223) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6225) vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6226) ret = enter_vmx_operation(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6227) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6228) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6230) /* Empty 'VMXON' state is permitted if no VMCS loaded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6231) if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6232) /* See vmx_has_valid_vmcs12. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6233) if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6234) (kvm_state->flags & KVM_STATE_NESTED_EVMCS) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6235) (kvm_state->hdr.vmx.vmcs12_pa != -1ull))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6236) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6237) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6241) if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6242) if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6243) !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6244) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6246) set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6247) } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6249) * nested_vmx_handle_enlightened_vmptrld() cannot be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6250) * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6251) * restored yet. EVMCS will be mapped from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6252) * nested_get_vmcs12_pages().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6254) kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6255) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6256) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6259) if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6260) vmx->nested.smm.vmxon = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6261) vmx->nested.vmxon = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6263) if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6264) vmx->nested.smm.guest_mode = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6267) vmcs12 = get_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6268) if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6269) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6271) if (vmcs12->hdr.revision_id != VMCS12_REVISION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6272) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6274) if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6275) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6277) vmx->nested.nested_run_pending =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6278) !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6280) vmx->nested.mtf_pending =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6281) !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6283) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6284) if (nested_cpu_has_shadow_vmcs(vmcs12) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6285) vmcs12->vmcs_link_pointer != -1ull) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6286) struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6288) if (kvm_state->size <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6289) sizeof(*kvm_state) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6290) sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6291) goto error_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6293) if (copy_from_user(shadow_vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6294) user_vmx_nested_state->shadow_vmcs12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6295) sizeof(*shadow_vmcs12))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6296) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6297) goto error_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6300) if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6301) !shadow_vmcs12->hdr.shadow_vmcs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6302) goto error_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6305) vmx->nested.has_preemption_timer_deadline = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6306) if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6307) vmx->nested.has_preemption_timer_deadline = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6308) vmx->nested.preemption_timer_deadline =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6309) kvm_state->hdr.vmx.preemption_timer_deadline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6312) if (nested_vmx_check_controls(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6313) nested_vmx_check_host_state(vcpu, vmcs12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6314) nested_vmx_check_guest_state(vcpu, vmcs12, &ignored))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6315) goto error_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6317) vmx->nested.dirty_vmcs12 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6318) ret = nested_vmx_enter_non_root_mode(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6319) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6320) goto error_guest_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6322) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6324) error_guest_mode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6325) vmx->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6326) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6329) void nested_vmx_set_vmcs_shadowing_bitmap(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6331) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6332) vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6333) vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6338) * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6339) * returned for the various VMX controls MSRs when nested VMX is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6340) * The same values should also be used to verify that vmcs12 control fields are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6341) * valid during nested entry from L1 to L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6342) * Each of these control msrs has a low and high 32-bit half: A low bit is on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6343) * if the corresponding bit in the (32-bit) control field *must* be on, and a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6344) * bit in the high half is on if the corresponding bit in the control field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6345) * may be on. See also vmx_control_verify().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6346) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6347) void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6349) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6350) * Note that as a general rule, the high half of the MSRs (bits in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6351) * the control fields which may be 1) should be initialized by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6352) * intersection of the underlying hardware's MSR (i.e., features which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6353) * can be supported) and the list of features we want to expose -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6354) * because they are known to be properly supported in our code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6355) * Also, usually, the low half of the MSRs (bits which must be 1) can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6356) * be set to 0, meaning that L1 may turn off any of these bits. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6357) * reason is that if one of these bits is necessary, it will appear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6358) * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6359) * fields of vmcs01 and vmcs02, will turn these bits off - and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6360) * nested_vmx_l1_wants_exit() will not pass related exits to L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6361) * These rules have exceptions below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6362) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6364) /* pin-based controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6365) rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6366) msrs->pinbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6367) msrs->pinbased_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6368) msrs->pinbased_ctls_low |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6369) PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6370) msrs->pinbased_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6371) PIN_BASED_EXT_INTR_MASK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6372) PIN_BASED_NMI_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6373) PIN_BASED_VIRTUAL_NMIS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6374) (enable_apicv ? PIN_BASED_POSTED_INTR : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6375) msrs->pinbased_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6376) PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6377) PIN_BASED_VMX_PREEMPTION_TIMER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6379) /* exit controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6380) rdmsr(MSR_IA32_VMX_EXIT_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6381) msrs->exit_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6382) msrs->exit_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6383) msrs->exit_ctls_low =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6384) VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6386) msrs->exit_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6387) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6388) VM_EXIT_HOST_ADDR_SPACE_SIZE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6389) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6390) VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6391) VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6392) msrs->exit_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6393) VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6394) VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6395) VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6397) /* We support free control of debug control saving. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6398) msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6400) /* entry controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6401) rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6402) msrs->entry_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6403) msrs->entry_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6404) msrs->entry_ctls_low =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6405) VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6406) msrs->entry_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6407) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6408) VM_ENTRY_IA32E_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6409) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6410) VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6411) VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6412) msrs->entry_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6413) (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6415) /* We support free control of debug control loading. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6416) msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6418) /* cpu-based controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6419) rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6420) msrs->procbased_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6421) msrs->procbased_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6422) msrs->procbased_ctls_low =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6423) CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6424) msrs->procbased_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6425) CPU_BASED_INTR_WINDOW_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6426) CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6427) CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6428) CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6429) CPU_BASED_CR3_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6430) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6431) CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6432) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6433) CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6434) CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6435) CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6436) CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6437) CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6439) * We can allow some features even when not supported by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6440) * hardware. For example, L1 can specify an MSR bitmap - and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6441) * can use it to avoid exits to L1 - even when L0 runs L2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6442) * without MSR bitmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6443) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6444) msrs->procbased_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6445) CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6446) CPU_BASED_USE_MSR_BITMAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6448) /* We support free control of CR3 access interception. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6449) msrs->procbased_ctls_low &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6450) ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6452) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6453) * secondary cpu-based controls. Do not include those that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6454) * depend on CPUID bits, they are added later by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6455) * vmx_vcpu_after_set_cpuid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6457) if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6458) rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6459) msrs->secondary_ctls_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6460) msrs->secondary_ctls_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6462) msrs->secondary_ctls_low = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6463) msrs->secondary_ctls_high &=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6464) SECONDARY_EXEC_DESC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6465) SECONDARY_EXEC_ENABLE_RDTSCP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6466) SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6467) SECONDARY_EXEC_WBINVD_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6468) SECONDARY_EXEC_APIC_REGISTER_VIRT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6469) SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6470) SECONDARY_EXEC_RDRAND_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6471) SECONDARY_EXEC_ENABLE_INVPCID |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6472) SECONDARY_EXEC_RDSEED_EXITING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6473) SECONDARY_EXEC_XSAVES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6475) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6476) * We can emulate "VMCS shadowing," even if the hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6477) * doesn't support it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6478) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6479) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6480) SECONDARY_EXEC_SHADOW_VMCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6482) if (enable_ept) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6483) /* nested EPT: emulate EPT also to L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6484) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6485) SECONDARY_EXEC_ENABLE_EPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6486) msrs->ept_caps =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6487) VMX_EPT_PAGE_WALK_4_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6488) VMX_EPT_PAGE_WALK_5_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6489) VMX_EPTP_WB_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6490) VMX_EPT_INVEPT_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6491) VMX_EPT_EXECUTE_ONLY_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6493) msrs->ept_caps &= ept_caps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6494) msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6495) VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6496) VMX_EPT_1GB_PAGE_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6497) if (enable_ept_ad_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6498) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6499) SECONDARY_EXEC_ENABLE_PML;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6500) msrs->ept_caps |= VMX_EPT_AD_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6504) if (cpu_has_vmx_vmfunc()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6505) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6506) SECONDARY_EXEC_ENABLE_VMFUNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6507) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6508) * Advertise EPTP switching unconditionally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6509) * since we emulate it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6511) if (enable_ept)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6512) msrs->vmfunc_controls =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6513) VMX_VMFUNC_EPTP_SWITCHING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6516) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6517) * Old versions of KVM use the single-context version without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6518) * checking for support, so declare that it is supported even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6519) * though it is treated as global context. The alternative is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6520) * not failing the single-context invvpid, and it is worse.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6522) if (enable_vpid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6523) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6524) SECONDARY_EXEC_ENABLE_VPID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6525) msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6526) VMX_VPID_EXTENT_SUPPORTED_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6529) if (enable_unrestricted_guest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6530) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6531) SECONDARY_EXEC_UNRESTRICTED_GUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6533) if (flexpriority_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6534) msrs->secondary_ctls_high |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6535) SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6537) /* miscellaneous data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6538) rdmsr(MSR_IA32_VMX_MISC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6539) msrs->misc_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6540) msrs->misc_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6541) msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6542) msrs->misc_low |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6543) MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6544) VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6545) VMX_MISC_ACTIVITY_HLT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6546) msrs->misc_high = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6548) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6549) * This MSR reports some information about VMX support. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6550) * should return information about the VMX we emulate for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6551) * guest, and the VMCS structure we give it - not about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6552) * VMX support of the underlying hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6554) msrs->basic =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6555) VMCS12_REVISION |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6556) VMX_BASIC_TRUE_CTLS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6557) ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6558) (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6560) if (cpu_has_vmx_basic_inout())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6561) msrs->basic |= VMX_BASIC_INOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6563) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6564) * These MSRs specify bits which the guest must keep fixed on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6565) * while L1 is in VMXON mode (in L1's root mode, or running an L2).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6566) * We picked the standard core2 setting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6567) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6568) #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6569) #define VMXON_CR4_ALWAYSON X86_CR4_VMXE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6570) msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6571) msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6573) /* These MSRs specify bits which the guest must keep fixed off. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6574) rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6575) rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6577) /* highest index: VMX_PREEMPTION_TIMER_VALUE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6578) msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6581) void nested_vmx_hardware_unsetup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6583) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6585) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6586) for (i = 0; i < VMX_BITMAP_NR; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6587) free_page((unsigned long)vmx_bitmap[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6591) __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6593) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6595) if (!cpu_has_vmx_shadow_vmcs())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6596) enable_shadow_vmcs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6597) if (enable_shadow_vmcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6598) for (i = 0; i < VMX_BITMAP_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6599) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6600) * The vmx_bitmap is not tied to a VM and so should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6601) * not be charged to a memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6602) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6603) vmx_bitmap[i] = (unsigned long *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6604) __get_free_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6605) if (!vmx_bitmap[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6606) nested_vmx_hardware_unsetup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6607) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6611) init_vmcs_shadow_fields();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6614) exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6615) exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6616) exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6617) exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6618) exit_handlers[EXIT_REASON_VMREAD] = handle_vmread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6619) exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6620) exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6621) exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6622) exit_handlers[EXIT_REASON_VMON] = handle_vmon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6623) exit_handlers[EXIT_REASON_INVEPT] = handle_invept;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6624) exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6625) exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6627) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6630) struct kvm_x86_nested_ops vmx_nested_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6631) .leave_nested = vmx_leave_nested,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6632) .check_events = vmx_check_nested_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6633) .hv_timer_pending = nested_vmx_preemption_timer_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6634) .get_state = vmx_get_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6635) .set_state = vmx_set_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6636) .get_nested_state_pages = vmx_get_nested_state_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6637) .write_log_dirty = nested_vmx_write_pml_buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6638) .enable_evmcs = nested_enable_evmcs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6639) .get_evmcs_version = nested_get_evmcs_version,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6640) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags