^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel-based Virtual Machine driver for Linux
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * AMD SVM support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2006 Qumranet, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright 2010 Red Hat, Inc. and/or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Yaniv Kamay <yaniv@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Avi Kivity <avi@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #define pr_fmt(fmt) "SVM: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kvm_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/msr-index.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/debugreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "kvm_emulate.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include "mmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include "cpuid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include "lapic.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include "svm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct x86_exception *fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * TODO: track the cause of the nested page fault, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * correctly fill in the high bits of exit_info_1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) svm->vmcb->control.exit_code = SVM_EXIT_NPF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) svm->vmcb->control.exit_code_hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) svm->vmcb->control.exit_info_1 = (1ULL << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) svm->vmcb->control.exit_info_2 = fault->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) svm->vmcb->control.exit_info_1 |= fault->error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) WARN_ON(!is_guest_mode(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) !svm->nested.nested_run_pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) svm->vmcb->control.exit_code_hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) svm->vmcb->control.exit_info_1 = fault->error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) svm->vmcb->control.exit_info_2 = fault->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) kvm_inject_page_fault(vcpu, fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) u64 cr3 = svm->nested.ctl.nested_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) u64 pdpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) offset_in_page(cr3) + index * 8, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return pdpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return svm->nested.ctl.nested_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct vmcb *hsave = svm->nested.hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) WARN_ON(mmu_is_nested(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) vcpu->arch.mmu = &vcpu->arch.guest_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, hsave->save.cr4, hsave->save.efer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) svm->nested.ctl.nested_cr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) vcpu->arch.mmu = &vcpu->arch.root_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) void recalc_intercepts(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct vmcb_control_area *c, *h, *g;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (!is_guest_mode(&svm->vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) c = &svm->vmcb->control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) h = &svm->nested.hsave->control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) g = &svm->nested.ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) for (i = 0; i < MAX_INTERCEPT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) c->intercepts[i] = h->intercepts[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (g->int_ctl & V_INTR_MASKING_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* We only want the cr8 intercept bits of L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) vmcb_clr_intercept(c, INTERCEPT_CR8_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * affect any interrupt we may want to inject; therefore,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * interrupt window vmexits are irrelevant to L0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) vmcb_clr_intercept(c, INTERCEPT_VINTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /* We don't want to see VMMCALLs from a nested guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) vmcb_clr_intercept(c, INTERCEPT_VMMCALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) for (i = 0; i < MAX_INTERCEPT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) c->intercepts[i] |= g->intercepts[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) vmcb_set_intercept(c, INTERCEPT_VMLOAD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) vmcb_set_intercept(c, INTERCEPT_VMSAVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static void copy_vmcb_control_area(struct vmcb_control_area *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) struct vmcb_control_area *from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) for (i = 0; i < MAX_INTERCEPT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) dst->intercepts[i] = from->intercepts[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) dst->iopm_base_pa = from->iopm_base_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) dst->msrpm_base_pa = from->msrpm_base_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) dst->tsc_offset = from->tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /* asid not copied, it is handled manually for svm->vmcb. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) dst->tlb_ctl = from->tlb_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) dst->int_ctl = from->int_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) dst->int_vector = from->int_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) dst->int_state = from->int_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) dst->exit_code = from->exit_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) dst->exit_code_hi = from->exit_code_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) dst->exit_info_1 = from->exit_info_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) dst->exit_info_2 = from->exit_info_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) dst->exit_int_info = from->exit_int_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) dst->exit_int_info_err = from->exit_int_info_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) dst->nested_ctl = from->nested_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) dst->event_inj = from->event_inj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) dst->event_inj_err = from->event_inj_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) dst->nested_cr3 = from->nested_cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) dst->virt_ext = from->virt_ext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) dst->pause_filter_count = from->pause_filter_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dst->pause_filter_thresh = from->pause_filter_thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * This function merges the msr permission bitmaps of kvm and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * nested vmcb. It is optimized in that it only merges the parts where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * the kvm msr permission bitmap may contain zero bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) for (i = 0; i < MSRPM_OFFSETS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) u32 value, p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) u64 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (msrpm_offsets[i] == 0xffffffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) p = msrpm_offsets[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) offset = svm->nested.ctl.msrpm_base_pa + (p * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) svm->nested.msrpm[p] = svm->msrpm[p] | value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (WARN_ON(!is_guest_mode(vcpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (!nested_svm_vmrun_msrpm(svm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) vcpu->run->internal.suberror =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) KVM_INTERNAL_ERROR_EMULATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) vcpu->run->internal.ndata = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static bool nested_vmcb_check_controls(struct vmcb_control_area *control)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if ((vmcb_is_intercept(control, INTERCEPT_VMRUN)) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (control->asid == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if ((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) !npt_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static bool nested_vmcb_check_save(struct vcpu_svm *svm, struct vmcb *vmcb12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct kvm_vcpu *vcpu = &svm->vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) bool vmcb12_lma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * FIXME: these should be done after copying the fields,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * to avoid TOC/TOU races. For these save area checks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * the possible damage is limited since kvm_set_cr0 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * kvm_set_cr4 handle failure; EFER_SVME is an exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * so it is force-set later in nested_prepare_vmcb_save.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if ((vmcb12->save.efer & EFER_SVME) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (((vmcb12->save.cr0 & X86_CR0_CD) == 0) && (vmcb12->save.cr0 & X86_CR0_NW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (!kvm_dr6_valid(vmcb12->save.dr6) || !kvm_dr7_valid(vmcb12->save.dr7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) vmcb12_lma = (vmcb12->save.efer & EFER_LME) && (vmcb12->save.cr0 & X86_CR0_PG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (vmcb12_lma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (!(vmcb12->save.cr4 & X86_CR4_PAE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) !(vmcb12->save.cr0 & X86_CR0_PE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) (vmcb12->save.cr3 & vcpu->arch.cr3_lm_rsvd_bits))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (kvm_valid_cr4(&svm->vcpu, vmcb12->save.cr4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static void load_nested_vmcb_control(struct vcpu_svm *svm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct vmcb_control_area *control)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) copy_vmcb_control_area(&svm->nested.ctl, control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /* Copy it here because nested_svm_check_controls will check it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) svm->nested.ctl.asid = control->asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) svm->nested.ctl.iopm_base_pa &= ~0x0fffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * Synchronize fields that are written by the processor, so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * they can be copied back into the nested_vmcb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) void sync_nested_vmcb_control(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) svm->nested.ctl.event_inj = svm->vmcb->control.event_inj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Only a few fields of int_ctl are written by the processor. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) mask = V_IRQ_MASK | V_TPR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) svm_is_intercept(svm, INTERCEPT_VINTR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * In order to request an interrupt window, L0 is usurping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * svm->vmcb->control.int_ctl and possibly setting V_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * even if it was clear in L1's VMCB. Restoring it would be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * wrong. However, in this case V_IRQ will remain true until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * interrupt_window_interception calls svm_clear_vintr and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * restores int_ctl. We can just leave it aside.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) mask &= ~V_IRQ_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) svm->nested.ctl.int_ctl &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * Transfer any event that L0 or L1 wanted to inject into L2 to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * EXIT_INT_INFO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static void nested_vmcb_save_pending_event(struct vcpu_svm *svm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) struct vmcb *vmcb12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct kvm_vcpu *vcpu = &svm->vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) u32 exit_int_info = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) unsigned int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (vcpu->arch.exception.injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) nr = vcpu->arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (vcpu->arch.exception.has_error_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) exit_int_info |= SVM_EVTINJ_VALID_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) vmcb12->control.exit_int_info_err =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) vcpu->arch.exception.error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) } else if (vcpu->arch.nmi_injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) } else if (vcpu->arch.interrupt.injected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) nr = vcpu->arch.interrupt.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) exit_int_info = nr | SVM_EVTINJ_VALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (vcpu->arch.interrupt.soft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) exit_int_info |= SVM_EVTINJ_TYPE_SOFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) exit_int_info |= SVM_EVTINJ_TYPE_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) vmcb12->control.exit_int_info = exit_int_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static inline bool nested_npt_enabled(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * if we are emulating VM-Entry into a guest with NPT enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) bool nested_npt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (cr3 & rsvd_bits(cpuid_maxphyaddr(vcpu), 63))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (!nested_npt && is_pae_paging(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * TODO: optimize unconditional TLB flush/MMU sync here and in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * kvm_init_shadow_npt_mmu().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (!nested_npt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) kvm_mmu_new_pgd(vcpu, cr3, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) vcpu->arch.cr3 = cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) kvm_init_mmu(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *vmcb12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /* Load the nested guest state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) svm->vmcb->save.es = vmcb12->save.es;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) svm->vmcb->save.cs = vmcb12->save.cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) svm->vmcb->save.ss = vmcb12->save.ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) svm->vmcb->save.ds = vmcb12->save.ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) svm->vmcb->save.gdtr = vmcb12->save.gdtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) svm->vmcb->save.idtr = vmcb12->save.idtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * Force-set EFER_SVME even though it is checked earlier on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * VMCB12, because the guest can flip the bit between the check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * and now. Clearing EFER_SVME would call svm_free_nested.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) svm_set_efer(&svm->vcpu, vmcb12->save.efer | EFER_SVME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) svm_set_cr0(&svm->vcpu, vmcb12->save.cr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) svm_set_cr4(&svm->vcpu, vmcb12->save.cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = vmcb12->save.cr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) kvm_rax_write(&svm->vcpu, vmcb12->save.rax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) kvm_rip_write(&svm->vcpu, vmcb12->save.rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* In case we don't even reach vcpu_run, the fields are not updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) svm->vmcb->save.rax = vmcb12->save.rax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) svm->vmcb->save.rsp = vmcb12->save.rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) svm->vmcb->save.rip = vmcb12->save.rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) svm->vmcb->save.dr7 = vmcb12->save.dr7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) svm->vcpu.arch.dr6 = vmcb12->save.dr6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) svm->vmcb->save.cpl = vmcb12->save.cpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static void nested_prepare_vmcb_control(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) const u32 int_ctl_vmcb01_bits =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (nested_npt_enabled(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) nested_svm_init_mmu_context(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) svm->vmcb->control.int_ctl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) (svm->nested.hsave->control.int_ctl & int_ctl_vmcb01_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) svm->vmcb->control.int_state = svm->nested.ctl.int_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) svm->vmcb->control.event_inj = svm->nested.ctl.event_inj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* Enter Guest-Mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) enter_guest_mode(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * Merge guest and host intercepts - must be called with vcpu in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * guest-mode to take affect here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) recalc_intercepts(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) vmcb_mark_all_dirty(svm->vmcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) int enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb12_gpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) struct vmcb *vmcb12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) svm->nested.vmcb12_gpa = vmcb12_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) nested_prepare_vmcb_save(svm, vmcb12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) nested_prepare_vmcb_control(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) nested_npt_enabled(svm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (!npt_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) svm->vcpu.arch.mmu->inject_page_fault = svm_inject_page_fault_nested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) svm_set_gif(svm, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) int nested_svm_vmrun(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct vmcb *vmcb12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct vmcb *hsave = svm->nested.hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct vmcb *vmcb = svm->vmcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct kvm_host_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) u64 vmcb12_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (is_smm(&svm->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) kvm_queue_exception(&svm->vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) vmcb12_gpa = svm->vmcb->save.rax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb12_gpa), &map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (ret == -EINVAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) kvm_inject_gp(&svm->vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) } else if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) return kvm_skip_emulated_instruction(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ret = kvm_skip_emulated_instruction(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) vmcb12 = map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (WARN_ON_ONCE(!svm->nested.initialized))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) load_nested_vmcb_control(svm, &vmcb12->control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (!nested_vmcb_check_save(svm, vmcb12) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) !nested_vmcb_check_controls(&svm->nested.ctl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) vmcb12->control.exit_code = SVM_EXIT_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) vmcb12->control.exit_code_hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) vmcb12->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) vmcb12->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) vmcb12->save.rip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) vmcb12->control.int_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) vmcb12->control.event_inj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) vmcb12->control.nested_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) vmcb12->control.intercepts[INTERCEPT_CR] >> 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) vmcb12->control.intercepts[INTERCEPT_EXCEPTION],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) vmcb12->control.intercepts[INTERCEPT_WORD3],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) vmcb12->control.intercepts[INTERCEPT_WORD4],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) vmcb12->control.intercepts[INTERCEPT_WORD5]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /* Clear internal status */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) kvm_clear_exception_queue(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) kvm_clear_interrupt_queue(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * Save the old vmcb, so we don't need to pick what we save, but can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * restore everything when a VMEXIT occurs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) hsave->save.es = vmcb->save.es;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) hsave->save.cs = vmcb->save.cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) hsave->save.ss = vmcb->save.ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) hsave->save.ds = vmcb->save.ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) hsave->save.gdtr = vmcb->save.gdtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) hsave->save.idtr = vmcb->save.idtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) hsave->save.efer = svm->vcpu.arch.efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) hsave->save.cr0 = kvm_read_cr0(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) hsave->save.cr4 = svm->vcpu.arch.cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) hsave->save.rip = kvm_rip_read(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) hsave->save.rsp = vmcb->save.rsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) hsave->save.rax = vmcb->save.rax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (npt_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) hsave->save.cr3 = vmcb->save.cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) hsave->save.cr3 = kvm_read_cr3(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) copy_vmcb_control_area(&hsave->control, &vmcb->control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) svm->nested.nested_run_pending = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (enter_svm_guest_mode(svm, vmcb12_gpa, vmcb12))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) goto out_exit_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (nested_svm_vmrun_msrpm(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) out_exit_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) svm->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) svm->vmcb->control.exit_code = SVM_EXIT_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) svm->vmcb->control.exit_code_hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) svm->vmcb->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) svm->vmcb->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) kvm_vcpu_unmap(&svm->vcpu, &map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) to_vmcb->save.fs = from_vmcb->save.fs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) to_vmcb->save.gs = from_vmcb->save.gs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) to_vmcb->save.tr = from_vmcb->save.tr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) to_vmcb->save.ldtr = from_vmcb->save.ldtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) to_vmcb->save.star = from_vmcb->save.star;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) to_vmcb->save.lstar = from_vmcb->save.lstar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) to_vmcb->save.cstar = from_vmcb->save.cstar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) to_vmcb->save.sfmask = from_vmcb->save.sfmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) int nested_svm_vmexit(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct vmcb *vmcb12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) struct vmcb *hsave = svm->nested.hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) struct vmcb *vmcb = svm->vmcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) struct kvm_host_map map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (rc == -EINVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) kvm_inject_gp(&svm->vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) vmcb12 = map.hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /* Exit Guest-Mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) leave_guest_mode(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) svm->nested.vmcb12_gpa = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) WARN_ON_ONCE(svm->nested.nested_run_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, &svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) /* in case we halted in L2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /* Give the current vmcb to the guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) vmcb12->save.es = vmcb->save.es;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) vmcb12->save.cs = vmcb->save.cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) vmcb12->save.ss = vmcb->save.ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) vmcb12->save.ds = vmcb->save.ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) vmcb12->save.gdtr = vmcb->save.gdtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) vmcb12->save.idtr = vmcb->save.idtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) vmcb12->save.efer = svm->vcpu.arch.efer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) vmcb12->save.cr0 = kvm_read_cr0(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) vmcb12->save.cr3 = kvm_read_cr3(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) vmcb12->save.cr2 = vmcb->save.cr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) vmcb12->save.cr4 = svm->vcpu.arch.cr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) vmcb12->save.rflags = kvm_get_rflags(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) vmcb12->save.rip = kvm_rip_read(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) vmcb12->save.rsp = kvm_rsp_read(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) vmcb12->save.rax = kvm_rax_read(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) vmcb12->save.dr7 = vmcb->save.dr7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) vmcb12->save.dr6 = svm->vcpu.arch.dr6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) vmcb12->save.cpl = vmcb->save.cpl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) vmcb12->control.int_state = vmcb->control.int_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) vmcb12->control.exit_code = vmcb->control.exit_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) vmcb12->control.exit_info_1 = vmcb->control.exit_info_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) vmcb12->control.exit_info_2 = vmcb->control.exit_info_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (vmcb12->control.exit_code != SVM_EXIT_ERR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) nested_vmcb_save_pending_event(svm, vmcb12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (svm->nrips_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) vmcb12->control.next_rip = vmcb->control.next_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) vmcb12->control.int_ctl = svm->nested.ctl.int_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) vmcb12->control.event_inj = svm->nested.ctl.event_inj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) vmcb12->control.pause_filter_count =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) svm->vmcb->control.pause_filter_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) vmcb12->control.pause_filter_thresh =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) svm->vmcb->control.pause_filter_thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) /* Restore the original control entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) copy_vmcb_control_area(&vmcb->control, &hsave->control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /* On vmexit the GIF is set to false */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) svm_set_gif(svm, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) svm->vcpu.arch.l1_tsc_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) svm->nested.ctl.nested_cr3 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /* Restore selected save entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) svm->vmcb->save.es = hsave->save.es;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) svm->vmcb->save.cs = hsave->save.cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) svm->vmcb->save.ss = hsave->save.ss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) svm->vmcb->save.ds = hsave->save.ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) svm->vmcb->save.gdtr = hsave->save.gdtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) svm->vmcb->save.idtr = hsave->save.idtr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) svm_set_efer(&svm->vcpu, hsave->save.efer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) svm_set_cr4(&svm->vcpu, hsave->save.cr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) kvm_rax_write(&svm->vcpu, hsave->save.rax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) kvm_rsp_write(&svm->vcpu, hsave->save.rsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) kvm_rip_write(&svm->vcpu, hsave->save.rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) svm->vmcb->save.dr7 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) svm->vmcb->save.cpl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) svm->vmcb->control.exit_int_info = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) vmcb_mark_all_dirty(svm->vmcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) vmcb12->control.exit_info_1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) vmcb12->control.exit_info_2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) vmcb12->control.exit_int_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) vmcb12->control.exit_int_info_err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) KVM_ISA_SVM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) kvm_vcpu_unmap(&svm->vcpu, &map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) nested_svm_uninit_mmu_context(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) rc = nested_svm_load_cr3(&svm->vcpu, hsave->save.cr3, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (npt_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) svm->vmcb->save.cr3 = hsave->save.cr3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) * Drop what we picked up for L2 via svm_complete_interrupts() so it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * doesn't end up in L1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) svm->vcpu.arch.nmi_injected = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) kvm_clear_exception_queue(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) kvm_clear_interrupt_queue(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) int svm_allocate_nested(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) struct page *hsave_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) if (svm->nested.initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) hsave_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (!hsave_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) svm->nested.hsave = page_address(hsave_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) svm->nested.msrpm = svm_vcpu_alloc_msrpm();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (!svm->nested.msrpm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) goto err_free_hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) svm->nested.initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) err_free_hsave:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) __free_page(hsave_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) void svm_free_nested(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (!svm->nested.initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) svm_vcpu_free_msrpm(svm->nested.msrpm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) svm->nested.msrpm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) __free_page(virt_to_page(svm->nested.hsave));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) svm->nested.hsave = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) svm->nested.initialized = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * Forcibly leave nested mode in order to be able to reset the VCPU later on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) void svm_leave_nested(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (is_guest_mode(&svm->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct vmcb *hsave = svm->nested.hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct vmcb *vmcb = svm->vmcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) svm->nested.nested_run_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) leave_guest_mode(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) copy_vmcb_control_area(&vmcb->control, &hsave->control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) nested_svm_uninit_mmu_context(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, &svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) u32 offset, msr, value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) int write, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) msr = svm->vcpu.arch.regs[VCPU_REGS_RCX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) offset = svm_msrpm_offset(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) write = svm->vmcb->control.exit_info_1 & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) mask = 1 << ((2 * (msr & 0xf)) + write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (offset == MSR_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) return NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) /* Offset is in 32 bit units but need in 8 bit units */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) offset *= 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) unsigned port, size, iopm_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) u16 val, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) u8 start_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) u64 gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (!(vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) return NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) port = svm->vmcb->control.exit_info_1 >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) SVM_IOIO_SIZE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) gpa = svm->nested.ctl.iopm_base_pa + (port / 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) start_bit = port % 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) iopm_len = (start_bit + size > 8) ? 2 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) mask = (0xf >> (4 - size)) << start_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) return NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) static int nested_svm_intercept(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) u32 exit_code = svm->vmcb->control.exit_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) int vmexit = NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) switch (exit_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) case SVM_EXIT_MSR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) vmexit = nested_svm_exit_handled_msr(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) case SVM_EXIT_IOIO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) vmexit = nested_svm_intercept_ioio(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (vmcb_is_intercept(&svm->nested.ctl, exit_code))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) vmexit = NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (vmcb_is_intercept(&svm->nested.ctl, exit_code))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) vmexit = NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * Host-intercepted exceptions have been checked already in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * nested_svm_exit_special. There is nothing to do here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * the vmexit is injected by svm_check_nested_events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) vmexit = NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) case SVM_EXIT_ERR: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) vmexit = NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) default: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (vmcb_is_intercept(&svm->nested.ctl, exit_code))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) vmexit = NESTED_EXIT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) return vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) int nested_svm_exit_handled(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) int vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) vmexit = nested_svm_intercept(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (vmexit == NESTED_EXIT_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return vmexit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) int nested_svm_check_permissions(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (!(svm->vcpu.arch.efer & EFER_SVME) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) !is_paging(&svm->vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) kvm_queue_exception(&svm->vcpu, UD_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (svm->vmcb->save.cpl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) kvm_inject_gp(&svm->vcpu, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) static bool nested_exit_on_exception(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) unsigned int nr = svm->vcpu.arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) unsigned int nr = svm->vcpu.arch.exception.nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) svm->vmcb->control.exit_code_hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (svm->vcpu.arch.exception.has_error_code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * EXITINFO2 is undefined for all exception intercepts other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * than #PF.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (nr == PF_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (svm->vcpu.arch.exception.nested_apf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) else if (svm->vcpu.arch.exception.has_payload)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) } else if (nr == DB_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) /* See inject_pending_event. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) kvm_deliver_exception_payload(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (svm->vcpu.arch.dr7 & DR7_GD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) svm->vcpu.arch.dr7 &= ~DR7_GD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) kvm_update_dr7(&svm->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) WARN_ON(svm->vcpu.arch.exception.has_payload);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) static void nested_svm_smi(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) svm->vmcb->control.exit_code = SVM_EXIT_SMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) svm->vmcb->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) svm->vmcb->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) static void nested_svm_nmi(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) svm->vmcb->control.exit_code = SVM_EXIT_NMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) svm->vmcb->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) svm->vmcb->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) static void nested_svm_intr(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) svm->vmcb->control.exit_code = SVM_EXIT_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) svm->vmcb->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) svm->vmcb->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) static inline bool nested_exit_on_init(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_INIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static void nested_svm_init(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) svm->vmcb->control.exit_code = SVM_EXIT_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) svm->vmcb->control.exit_info_1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) svm->vmcb->control.exit_info_2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) nested_svm_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) static int svm_check_nested_events(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) bool block_nested_events =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) struct kvm_lapic *apic = vcpu->arch.apic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (lapic_in_kernel(vcpu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) test_bit(KVM_APIC_INIT, &apic->pending_events)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (!nested_exit_on_init(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) nested_svm_init(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (vcpu->arch.exception.pending) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (!nested_exit_on_exception(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) nested_svm_inject_exception_vmexit(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) if (!nested_exit_on_smi(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) nested_svm_smi(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (!nested_exit_on_nmi(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) nested_svm_nmi(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (block_nested_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (!nested_exit_on_intr(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) nested_svm_intr(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) int nested_svm_exit_special(struct vcpu_svm *svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) u32 exit_code = svm->vmcb->control.exit_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) switch (exit_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) case SVM_EXIT_INTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) case SVM_EXIT_NMI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) case SVM_EXIT_NPF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) return NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (get_host_vmcb(svm)->control.intercepts[INTERCEPT_EXCEPTION] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) excp_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) svm->vcpu.arch.apf.host_apf_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /* Trap async PF even if not shadowing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return NESTED_EXIT_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) return NESTED_EXIT_CONTINUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) static int svm_get_nested_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) struct kvm_nested_state __user *user_kvm_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) u32 user_data_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) struct vcpu_svm *svm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) struct kvm_nested_state kvm_state = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) .flags = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) .format = KVM_STATE_NESTED_FORMAT_SVM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) .size = sizeof(kvm_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) struct vmcb __user *user_vmcb = (struct vmcb __user *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) &user_kvm_nested_state->data.svm[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) if (!vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (user_data_size < kvm_state.size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) /* First fill in the header and copy it out. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (is_guest_mode(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (svm->nested.nested_run_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (gif_set(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) kvm_state.flags |= KVM_STATE_NESTED_GIF_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (!is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * Copy over the full size of the VMCB rather than just the size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * of the structs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (copy_to_user(&user_vmcb->control, &svm->nested.ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) sizeof(user_vmcb->control)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (copy_to_user(&user_vmcb->save, &svm->nested.hsave->save,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) sizeof(user_vmcb->save)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) return kvm_state.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) static int svm_set_nested_state(struct kvm_vcpu *vcpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct kvm_nested_state __user *user_kvm_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) struct kvm_nested_state *kvm_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct vcpu_svm *svm = to_svm(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) struct vmcb *hsave = svm->nested.hsave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) struct vmcb __user *user_vmcb = (struct vmcb __user *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) &user_kvm_nested_state->data.svm[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) struct vmcb_control_area *ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) struct vmcb_save_area *save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) u32 cr0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) KVM_STATE_NESTED_SVM_VMCB_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) KVM_STATE_NESTED_RUN_PENDING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) KVM_STATE_NESTED_GIF_SET))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) if (!(vcpu->arch.efer & EFER_SVME)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) /* GIF=1 and no guest mode are required if SVME=0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) /* SMM temporarily disables SVM, so we cannot be in guest mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) svm_leave_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) save = kzalloc(sizeof(*save), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) if (!ctl || !save)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) if (copy_from_user(save, &user_vmcb->save, sizeof(*save)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (!nested_vmcb_check_controls(ctl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * Processor state contains L2 state. Check that it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * valid for guest mode (see nested_vmcb_checks).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) cr0 = kvm_read_cr0(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * Validate host state saved from before VMRUN (see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * nested_svm_check_permissions).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) * TODO: validate reserved bits for all saved state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) if (!(save->cr0 & X86_CR0_PG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (!(save->efer & EFER_SVME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * All checks done, we can enter guest mode. L1 control fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * come from the nested save state. Guest state is already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * in the registers, the save area of the nested state instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * contains saved L1 state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) copy_vmcb_control_area(&hsave->control, &svm->vmcb->control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) hsave->save = *save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) if (is_guest_mode(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) svm_leave_nested(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) load_nested_vmcb_control(svm, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) nested_prepare_vmcb_control(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) kfree(save);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) kfree(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) struct kvm_x86_nested_ops svm_nested_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) .leave_nested = svm_leave_nested,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) .check_events = svm_check_nested_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) .get_nested_state_pages = svm_get_nested_state_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) .get_state = svm_get_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) .set_state = svm_set_nested_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags