^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * License. See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * TLB handlers run from KSEG0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Authors: Sanjay Lal <sanjayl@kymasys.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/srcu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/bootinfo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/mipsregs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/tlbdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #undef CONFIG_MIPS_MT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/r4kcache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define CONFIG_MIPS_MT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define KVM_GUEST_PC_TLB 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define KVM_GUEST_SP_TLB 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #ifdef CONFIG_KVM_MIPS_VZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) unsigned long GUESTID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) EXPORT_SYMBOL_GPL(GUESTID_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unsigned long GUESTID_FIRST_VERSION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) unsigned long GUESTID_VERSION_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (cpu_has_guestid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return cpu_asid(smp_processor_id(), gpa_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return cpu_asid(cpu, kern_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return cpu_asid(cpu, user_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* Structure defining an tlb entry data set. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) void kvm_mips_dump_host_tlbs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) kvm_info("HOST TLBs:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) dump_tlb_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) pr_info("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) dump_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct mips_coproc *cop0 = vcpu->arch.cop0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct kvm_mips_tlb tlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) kvm_info("Guest TLBs:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) tlb = vcpu->arch.guest_tlb[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) kvm_info("TLB%c%3d Hi 0x%08lx ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ? ' ' : '*',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) i, tlb.tlb_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) kvm_info("Lo0=0x%09llx %c%c attr %lx ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) tlb.tlb_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int index = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) TLB_HI_ASID_HIT(tlb[i], entryhi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) index = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int _kvm_mips_host_tlb_inv(unsigned long entryhi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) write_c0_entryhi(entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) tlb_probe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) tlb_probe_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) idx = read_c0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (idx >= current_cpu_data.tlbsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (idx >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) write_c0_entryhi(UNIQUE_ENTRYHI(idx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) write_c0_entrylo0(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) write_c0_entrylo1(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) bool user, bool kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * Initialize idx_user and idx_kernel to workaround bogus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * maybe-initialized warning when using GCC 6.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int idx_user = 0, idx_kernel = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned long flags, old_entryhi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) old_entryhi = read_c0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) idx_user = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) kvm_mips_get_user_asid(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) idx_kernel = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) kvm_mips_get_kernel_asid(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) write_c0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * We don't want to get reserved instruction exceptions for missing tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (cpu_has_vtag_icache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) flush_icache_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (user && idx_user >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) kvm_debug("%s: Invalidated guest user entryhi %#lx @ idx %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) __func__, (va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) kvm_mips_get_user_asid(vcpu), idx_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (kernel && idx_kernel >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) kvm_debug("%s: Invalidated guest kernel entryhi %#lx @ idx %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) __func__, (va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) kvm_mips_get_kernel_asid(vcpu), idx_kernel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #ifdef CONFIG_KVM_MIPS_VZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* GuestID management */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * clear_root_gid() - Set GuestCtl1.RID for normal root operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static inline void clear_root_gid(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (cpu_has_guestid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) clear_c0_guestctl1(MIPS_GCTL1_RID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * Sets the root GuestID to match the current guest GuestID, for TLB operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * on the GPA->RPA mappings in the root TLB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * The caller must be sure to disable HTW while the root GID is set, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * possibly longer if TLB registers are modified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static inline void set_root_gid_to_guest_gid(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) unsigned int guestctl1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (cpu_has_guestid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) guestctl1 = read_c0_guestctl1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) ((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) << MIPS_GCTL1_RID_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) write_c0_guestctl1(guestctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned long flags, old_entryhi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) htw_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) /* Set root GuestID for root probe and write of guest TLB entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) old_entryhi = read_c0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) kvm_mips_get_root_asid(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) write_c0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * We don't want to get reserved instruction exceptions for missing tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (cpu_has_vtag_icache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) flush_icache_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (idx > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) __func__, (va & VPN2_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) kvm_mips_get_root_asid(vcpu), idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * @vcpu: KVM VCPU pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * @gpa: Guest virtual address in a TLB mapped guest segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * @gpa: Ponter to output guest physical address it maps to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * Converts a guest virtual address in a guest TLB mapped segment to a guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * physical address, by probing the guest TLB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * Returns: 0 if guest TLB mapping exists for @gva. *@gpa will have been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * -EFAULT if no guest TLB mapping exists for @gva. *@gpa may not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * have been written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) unsigned long *gpa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) unsigned long o_entryhi, o_entrylo[2], o_pagemask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) unsigned int o_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) unsigned long entrylo[2], pagemask, pagemaskbit, pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /* Probe the guest TLB for a mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /* Set root GuestID for root probe of guest TLB entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) htw_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) o_entryhi = read_gc0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) o_index = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) guest_tlb_probe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) tlb_probe_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) index = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (index < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /* No match, fail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) write_gc0_entryhi(o_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) write_gc0_index(o_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /* Match! read the TLB entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) o_entrylo[0] = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) o_entrylo[1] = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) o_pagemask = read_gc0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) mtc0_tlbr_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) guest_tlb_read();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) tlb_read_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) entrylo[0] = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) entrylo[1] = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) pagemask = ~read_gc0_pagemask() & ~0x1fffl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) write_gc0_entryhi(o_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) write_gc0_index(o_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) write_gc0_entrylo0(o_entrylo[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) write_gc0_entrylo1(o_entrylo[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) write_gc0_pagemask(o_pagemask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /* Select one of the EntryLo values and interpret the GPA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) pa = entrylo[!!(gva & pagemaskbit)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * TLB entry may have become invalid since TLB probe if physical FTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * entries are shared between threads (e.g. I6400).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (!(pa & ENTRYLO_V))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * Note, this doesn't take guest MIPS32 XPA into account, where PFN is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * split with XI/RI in the middle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) pa = (pa << 6) & ~0xfffl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) pa |= gva & ~(pagemask | pagemaskbit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) *gpa = pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * guests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * Invalidate all entries in root tlb which are GPA mappings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) void kvm_vz_local_flush_roottlb_all_guests(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned long old_entryhi, old_pagemask, old_guestctl1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (WARN_ON(!cpu_has_guestid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) htw_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) old_entryhi = read_c0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) old_pagemask = read_c0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) old_guestctl1 = read_c0_guestctl1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * Invalidate guest entries in root TLB while leaving root entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * intact when possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) write_c0_index(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) tlb_read();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) tlb_read_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /* Don't invalidate non-guest (RVA) mappings in the root TLB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (!(read_c0_guestctl1() & MIPS_GCTL1_RID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) /* Make sure all entries differ. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) write_c0_entryhi(UNIQUE_ENTRYHI(entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) write_c0_entrylo0(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) write_c0_entrylo1(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) write_c0_guestctl1(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) write_c0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) write_c0_pagemask(old_pagemask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) write_c0_guestctl1(old_guestctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * Invalidate all entries in guest tlb irrespective of guestid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) void kvm_vz_local_flush_guesttlb_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) unsigned long old_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) unsigned long old_entryhi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) unsigned long old_entrylo[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) unsigned long old_pagemask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) int entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) u64 cvmmemctl2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* Preserve all clobbered guest registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) old_index = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) old_entryhi = read_gc0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) old_entrylo[0] = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) old_entrylo[1] = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) old_pagemask = read_gc0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) switch (current_cpu_type()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) case CPU_CAVIUM_OCTEON3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /* Inhibit machine check due to multiple matching TLB entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) cvmmemctl2 = read_c0_cvmmemctl2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) cvmmemctl2 |= CVMMEMCTL2_INHIBITTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) write_c0_cvmmemctl2(cvmmemctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /* Invalidate guest entries in guest TLB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) write_gc0_entrylo0(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) write_gc0_entrylo1(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) write_gc0_pagemask(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /* Make sure all entries differ. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) write_gc0_index(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) guest_tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (cvmmemctl2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) cvmmemctl2 &= ~CVMMEMCTL2_INHIBITTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) write_c0_cvmmemctl2(cvmmemctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) write_gc0_index(old_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) write_gc0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) write_gc0_entrylo0(old_entrylo[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) write_gc0_entrylo1(old_entrylo[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) write_gc0_pagemask(old_pagemask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * kvm_vz_save_guesttlb() - Save a range of guest TLB entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * @buf: Buffer to write TLB entries into.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * @index: Start index.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * @count: Number of entries to save.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * Save a range of guest TLB entries. The caller must ensure interrupts are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) unsigned int end = index + count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) unsigned int guestctl1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) int old_index, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Save registers we're about to clobber */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) old_index = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) old_entryhi = read_gc0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) old_entrylo0 = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) old_entrylo1 = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) old_pagemask = read_gc0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* Set root GuestID for root probe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) htw_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (cpu_has_guestid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) guestctl1 = read_c0_guestctl1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) /* Read each entry from guest TLB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) for (i = index; i < end; ++i, ++buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) write_gc0_index(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) mtc0_tlbr_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) guest_tlb_read();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) tlb_read_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (cpu_has_guestid &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) /* Entry invalid or belongs to another guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) buf->tlb_lo[0] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) buf->tlb_lo[1] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) buf->tlb_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) /* Entry belongs to the right guest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) buf->tlb_hi = read_gc0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) buf->tlb_lo[0] = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) buf->tlb_lo[1] = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) buf->tlb_mask = read_gc0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* Clear root GuestID again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /* Restore clobbered registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) write_gc0_index(old_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) write_gc0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) write_gc0_entrylo0(old_entrylo0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) write_gc0_entrylo1(old_entrylo1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) write_gc0_pagemask(old_pagemask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * kvm_vz_load_guesttlb() - Save a range of guest TLB entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * @buf: Buffer to read TLB entries from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * @index: Start index.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * @count: Number of entries to load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * Load a range of guest TLB entries. The caller must ensure interrupts are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) unsigned int end = index + count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) int old_index, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* Save registers we're about to clobber */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) old_index = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) old_entryhi = read_gc0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) old_entrylo0 = read_gc0_entrylo0();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) old_entrylo1 = read_gc0_entrylo1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) old_pagemask = read_gc0_pagemask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /* Set root GuestID for root probe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) htw_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) /* Write each entry to guest TLB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) for (i = index; i < end; ++i, ++buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) write_gc0_index(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) write_gc0_entryhi(buf->tlb_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) write_gc0_entrylo0(buf->tlb_lo[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) write_gc0_entrylo1(buf->tlb_lo[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) write_gc0_pagemask(buf->tlb_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) guest_tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /* Clear root GuestID again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) htw_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* Restore clobbered registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) write_gc0_index(old_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) write_gc0_entryhi(old_entryhi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) write_gc0_entrylo0(old_entrylo0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) write_gc0_entrylo1(old_entrylo1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) write_gc0_pagemask(old_pagemask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) #ifdef CONFIG_CPU_LOONGSON64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) void kvm_loongson_clear_guest_vtlb(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) int idx = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) /* Set root GuestID for root probe and write of guest TLB entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) write_gc0_index(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) guest_tlbinvf();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) write_gc0_index(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_vtlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) void kvm_loongson_clear_guest_ftlb(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) int idx = read_gc0_index();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) /* Set root GuestID for root probe and write of guest TLB entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) set_root_gid_to_guest_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) for (i = current_cpu_data.tlbsizevtlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) i < (current_cpu_data.tlbsizevtlb +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) current_cpu_data.tlbsizeftlbsets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) write_gc0_index(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) guest_tlbinvf();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) write_gc0_index(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) clear_root_gid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_ftlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * kvm_mips_suspend_mm() - Suspend the active mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * @cpu The CPU we're running on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * Suspend the active_mm, ready for a switch to a KVM guest virtual address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * space. This is left active for the duration of guest context, including time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * with interrupts enabled, so we need to be careful not to confuse e.g. cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * management IPIs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * kvm_mips_resume_mm() should be called before context switching to a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * process so we don't need to worry about reference counting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * This needs to be in static kernel code to avoid exporting init_mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) void kvm_mips_suspend_mm(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) cpumask_clear_cpu(cpu, mm_cpumask(current->active_mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) current->active_mm = &init_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) EXPORT_SYMBOL_GPL(kvm_mips_suspend_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) * kvm_mips_resume_mm() - Resume the current process mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * @cpu The CPU we're running on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) * Resume the mm of the current process, after a switch back from a KVM guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * virtual address space (see kvm_mips_suspend_mm()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) void kvm_mips_resume_mm(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) cpumask_set_cpu(cpu, mm_cpumask(current->mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) current->active_mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) EXPORT_SYMBOL_GPL(kvm_mips_resume_mm);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags