^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel-based Virtual Machine driver for Linux
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This module enables machines with Intel VT-x extensions to run virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * machines without emulation or binary translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2006 Qumranet, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright 2010 Red Hat, Inc. and/or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Avi Kivity <avi@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Yaniv Kamay <yaniv@qumranet.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <kvm/iodev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/kvm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/miscdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/reboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/syscore_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/sched/stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/anon_inodes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/kvm_para.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/srcu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/bsearch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/lockdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <asm/ioctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "coalesced_mmio.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "async_pf.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include "vfio.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <trace/events/kvm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Worst case buffer size needed for holding an integer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define ITOA_MAX_LEN 12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) MODULE_AUTHOR("Qumranet");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* Architectures should define their poll value according to the halt latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) module_param(halt_poll_ns, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) EXPORT_SYMBOL_GPL(halt_poll_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* Default doubles per-vcpu halt_poll_ns. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) unsigned int halt_poll_ns_grow = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) module_param(halt_poll_ns_grow, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /* The start value to grow halt_poll_ns from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) module_param(halt_poll_ns_grow_start, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /* Default resets per-vcpu halt_poll_ns . */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) unsigned int halt_poll_ns_shrink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) module_param(halt_poll_ns_shrink, uint, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * Ordering of locks:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * kvm->lock --> kvm->slots_lock --> kvm->irq_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) DEFINE_MUTEX(kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static DEFINE_RAW_SPINLOCK(kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) LIST_HEAD(vm_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static cpumask_var_t cpus_hardware_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static int kvm_usage_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static atomic_t hardware_enable_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static struct kmem_cache *kvm_vcpu_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static __read_mostly struct preempt_ops kvm_preempt_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct dentry *kvm_debugfs_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static int kvm_debugfs_num_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static const struct file_operations stat_fops_per_vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static struct file_operations kvm_chardev_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) unsigned long arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #ifdef CONFIG_KVM_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned long arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define KVM_COMPAT(c) .compat_ioctl = (c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * For architectures that don't implement a compat infrastructure,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * adopt a double line of defense:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * - Prevent a compat task from opening /dev/kvm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * - If the open has been done by a 64bit task, and the KVM fd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * passed to a compat task, let the ioctls fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) unsigned long arg) { return -EINVAL; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static int kvm_no_compat_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return is_compat_task() ? -ENODEV : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define KVM_COMPAT(c) .compat_ioctl = kvm_no_compat_ioctl, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) .open = kvm_no_compat_open
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static int hardware_enable_all(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) static void hardware_disable_all(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __visible bool kvm_rebooting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) EXPORT_SYMBOL_GPL(kvm_rebooting);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define KVM_EVENT_CREATE_VM 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define KVM_EVENT_DESTROY_VM 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static unsigned long long kvm_createvm_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static unsigned long long kvm_active_vms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) bool kvm_is_zone_device_pfn(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * The metadata used by is_zone_device_page() to determine whether or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * not a page is ZONE_DEVICE is guaranteed to be valid if and only if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * the device has been pinned, e.g. by get_user_pages(). WARN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * page_count() is zero to help detect bad usage of this helper.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (!pfn_valid(pfn) || WARN_ON_ONCE(!page_count(pfn_to_page(pfn))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return is_zone_device_page(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * ZONE_DEVICE pages currently set PG_reserved, but from a refcounting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * perspective they are "normal" pages, albeit with slightly different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * usage rules.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (pfn_valid(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return PageReserved(pfn_to_page(pfn)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) !is_zero_pfn(pfn) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) !kvm_is_zone_device_pfn(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) bool kvm_is_transparent_hugepage(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct page *page = pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (!PageTransCompoundMap(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return is_transparent_hugepage(compound_head(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * Switches to specified vcpu, until a matching vcpu_put()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) void vcpu_load(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) int cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) __this_cpu_write(kvm_running_vcpu, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) preempt_notifier_register(&vcpu->preempt_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) kvm_arch_vcpu_load(vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) EXPORT_SYMBOL_GPL(vcpu_load);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) void vcpu_put(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) kvm_arch_vcpu_put(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) preempt_notifier_unregister(&vcpu->preempt_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) __this_cpu_write(kvm_running_vcpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) EXPORT_SYMBOL_GPL(vcpu_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* TODO: merge with kvm_arch_vcpu_should_kick */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) int mode = kvm_vcpu_exiting_guest_mode(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * We need to wait for the VCPU to reenable interrupts and get out of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * READING_SHADOW_PAGE_TABLES mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (req & KVM_REQUEST_WAIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return mode != OUTSIDE_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * Need to kick a running VCPU, but otherwise there is nothing to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return mode == IN_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static void ack_flush(void *_completed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (unlikely(!cpus))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) cpus = cpu_online_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (cpumask_empty(cpus))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) smp_call_function_many(cpus, ack_flush, NULL, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct kvm_vcpu *except,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) unsigned long *vcpu_bitmap, cpumask_var_t tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) int i, cpu, me;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) bool called;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) me = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) kvm_for_each_vcpu(i, vcpu, kvm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if ((vcpu_bitmap && !test_bit(i, vcpu_bitmap)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) vcpu == except)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) kvm_make_request(req, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) cpu = vcpu->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) if (tmp != NULL && cpu != -1 && cpu != me &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) kvm_request_needs_ipi(vcpu, req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) __cpumask_set_cpu(cpu, tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return called;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) bool kvm_make_all_cpus_request_except(struct kvm *kvm, unsigned int req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct kvm_vcpu *except)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) cpumask_var_t cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) bool called;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) zalloc_cpumask_var(&cpus, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) called = kvm_make_vcpus_request_mask(kvm, req, except, NULL, cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) free_cpumask_var(cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return called;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return kvm_make_all_cpus_request_except(kvm, req, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) void kvm_flush_remote_tlbs(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * kvm_make_all_cpus_request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * We want to publish modifications to the page tables before reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * mode. Pairs with a memory barrier in arch-specific code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * and smp_mb in walk_shadow_page_lockless_begin/end.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * - powerpc: smp_mb in kvmppc_prepare_to_enter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * There is already an smp_mb__after_atomic() before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * barrier here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (!kvm_arch_flush_remote_tlb(kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ++kvm->stat.remote_tlb_flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) void kvm_reload_remote_mmus(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) gfp_t gfp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) gfp_flags |= mc->gfp_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (mc->kmem_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return kmem_cache_alloc(mc->kmem_cache, gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return (void *)__get_free_page(gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) void *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (mc->nobjs >= min)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) while (mc->nobjs < ARRAY_SIZE(mc->objects)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) obj = mmu_memory_cache_alloc_obj(mc, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return mc->nobjs >= min ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) mc->objects[mc->nobjs++] = obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return mc->nobjs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) while (mc->nobjs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (mc->kmem_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) free_page((unsigned long)mc->objects[--mc->nobjs]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (WARN_ON(!mc->nobjs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) p = mc->objects[--mc->nobjs];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) BUG_ON(!p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) mutex_init(&vcpu->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) vcpu->cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) vcpu->kvm = kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) vcpu->vcpu_id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) vcpu->pid = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) rcuwait_init(&vcpu->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) kvm_async_pf_vcpu_init(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) vcpu->pre_pcpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) kvm_vcpu_set_in_spin_loop(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) kvm_vcpu_set_dy_eligible(vcpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) vcpu->preempted = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) vcpu->ready = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) kvm_arch_vcpu_destroy(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * No need for rcu_read_lock as VCPU_RUN is the only place that changes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * the vcpu->pid pointer, and at destruction time all file descriptors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * are already gone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) put_pid(rcu_dereference_protected(vcpu->pid, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) free_page((unsigned long)vcpu->run);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) kmem_cache_free(kvm_vcpu_cache, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) EXPORT_SYMBOL_GPL(kvm_vcpu_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return container_of(mn, struct kvm, mmu_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static void kvm_mmu_notifier_invalidate_range(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) kvm_arch_mmu_notifier_invalidate_range(kvm, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) kvm->mmu_notifier_seq++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (kvm_set_spte_hva(kvm, address, pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) kvm_flush_remote_tlbs(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) const struct mmu_notifier_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) int need_tlb_flush = 0, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * The count increase must become visible at unlock time as no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * spte can be established without taking the mmu_lock and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * count is also read inside the mmu_lock critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) kvm->mmu_notifier_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) need_tlb_flush = kvm_unmap_hva_range(kvm, range->start, range->end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) range->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /* we've to flush the tlb before the pages can be freed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (need_tlb_flush || kvm->tlbs_dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) kvm_flush_remote_tlbs(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) const struct mmu_notifier_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * This sequence increase will notify the kvm page fault that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * the page that is going to be mapped in the spte could have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * been freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) kvm->mmu_notifier_seq++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * The above sequence increase must be visible before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * below count decrease, which is ensured by the smp_wmb above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * in conjunction with the smp_rmb in mmu_notifier_retry().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) kvm->mmu_notifier_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) BUG_ON(kvm->mmu_notifier_count < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) int young, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) young = kvm_age_hva(kvm, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (young)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) kvm_flush_remote_tlbs(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return young;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) int young, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * Even though we do not flush TLB, this will still adversely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * affect performance on pre-Haswell Intel EPT, where there is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * no EPT Access Bit to clear so that we have to tear down EPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * tables instead. If we find this unacceptable, we can always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * add a parameter to kvm_age_hva so that it effectively doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * do anything on clear_young.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * Also note that currently we never issue secondary TLB flushes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * from clear_young, leaving this job up to the regular system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * cadence. If we find this inaccurate, we might come up with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * more sophisticated heuristic later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) young = kvm_age_hva(kvm, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) return young;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) int young, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) young = kvm_test_age_hva(kvm, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return young;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct kvm *kvm = mmu_notifier_to_kvm(mn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) kvm_arch_flush_shadow_all(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) srcu_read_unlock(&kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) .invalidate_range = kvm_mmu_notifier_invalidate_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) .clear_flush_young = kvm_mmu_notifier_clear_flush_young,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) .clear_young = kvm_mmu_notifier_clear_young,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) .test_young = kvm_mmu_notifier_test_young,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) .change_pte = kvm_mmu_notifier_change_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) .release = kvm_mmu_notifier_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) static int kvm_init_mmu_notifier(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) #else /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) static int kvm_init_mmu_notifier(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) static struct kvm_memslots *kvm_alloc_memslots(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (!slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) slots->id_to_index[i] = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (!memslot->dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) kvfree(memslot->dirty_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) memslot->dirty_bitmap = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) kvm_destroy_dirty_bitmap(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) kvm_arch_free_memslot(kvm, slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) slot->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) slot->npages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) struct kvm_memory_slot *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (!slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) kvm_for_each_memslot(memslot, slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) kvm_free_memslot(kvm, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) static void kvm_destroy_vm_debugfs(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (!kvm->debugfs_dentry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) debugfs_remove_recursive(kvm->debugfs_dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (kvm->debugfs_stat_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) for (i = 0; i < kvm_debugfs_num_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) kfree(kvm->debugfs_stat_data[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) kfree(kvm->debugfs_stat_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) static DEFINE_MUTEX(kvm_debugfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct dentry *dent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) char dir_name[ITOA_MAX_LEN * 2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) struct kvm_stat_data *stat_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) struct kvm_stats_debugfs_item *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (!debugfs_initialized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) mutex_lock(&kvm_debugfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) dent = debugfs_lookup(dir_name, kvm_debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (dent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) dput(dent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) mutex_unlock(&kvm_debugfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) dent = debugfs_create_dir(dir_name, kvm_debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) mutex_unlock(&kvm_debugfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (IS_ERR(dent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) kvm->debugfs_dentry = dent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) sizeof(*kvm->debugfs_stat_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (!kvm->debugfs_stat_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) for (p = debugfs_entries; p->name; p++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (!stat_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) stat_data->kvm = kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) stat_data->dbgfs_item = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) kvm->debugfs_dentry, stat_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) &stat_fops_per_vm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * Called after the VM is otherwise initialized, but just before adding it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * the vm_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) int __weak kvm_arch_post_init_vm(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * Called just after removing the VM from the vm_list, but before doing any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * other destruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) static struct kvm *kvm_create_vm(unsigned long type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct kvm *kvm = kvm_arch_alloc_vm();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) int r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (!kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) spin_lock_init(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) mmgrab(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) kvm->mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) kvm_eventfd_init(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) mutex_init(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) mutex_init(&kvm->irq_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) mutex_init(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) INIT_LIST_HEAD(&kvm->devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (init_srcu_struct(&kvm->srcu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) goto out_err_no_srcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (init_srcu_struct(&kvm->irq_srcu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) goto out_err_no_irq_srcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) refcount_set(&kvm->users_count, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) struct kvm_memslots *slots = kvm_alloc_memslots();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (!slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) goto out_err_no_arch_destroy_vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) /* Generations must be different for each address space. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) slots->generation = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) rcu_assign_pointer(kvm->memslots[i], slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) for (i = 0; i < KVM_NR_BUSES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) rcu_assign_pointer(kvm->buses[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (!kvm->buses[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) goto out_err_no_arch_destroy_vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) kvm->max_halt_poll_ns = halt_poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) r = kvm_arch_init_vm(kvm, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) goto out_err_no_arch_destroy_vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) r = hardware_enable_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) goto out_err_no_disable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) #ifdef CONFIG_HAVE_KVM_IRQFD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) r = kvm_init_mmu_notifier(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) goto out_err_no_mmu_notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) r = kvm_arch_post_init_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) list_add(&kvm->vm_list, &vm_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) preempt_notifier_inc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) * When the fd passed to this ioctl() is opened it pins the module,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * but try_module_get() also prevents getting a reference if the module
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * is in MODULE_STATE_GOING (e.g. if someone ran "rmmod --wait").
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if (!try_module_get(kvm_chardev_ops.owner)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) r = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) return kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (kvm->mmu_notifier.ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) out_err_no_mmu_notifier:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) hardware_disable_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) out_err_no_disable:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) kvm_arch_destroy_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) out_err_no_arch_destroy_vm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) for (i = 0; i < KVM_NR_BUSES; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) kfree(kvm_get_bus(kvm, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) cleanup_srcu_struct(&kvm->irq_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) out_err_no_irq_srcu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) cleanup_srcu_struct(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) out_err_no_srcu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) kvm_arch_free_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) mmdrop(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return ERR_PTR(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) static void kvm_destroy_devices(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) struct kvm_device *dev, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * We do not need to take the kvm->lock here, because nobody else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * has a reference to the struct kvm at this point and therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * cannot access the devices list anyhow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) list_del(&dev->vm_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) dev->ops->destroy(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static void kvm_destroy_vm(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) struct mm_struct *mm = kvm->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) kvm_destroy_vm_debugfs(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) kvm_arch_sync_events(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) list_del(&kvm->vm_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) kvm_arch_pre_destroy_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) kvm_free_irq_routing(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) for (i = 0; i < KVM_NR_BUSES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) kvm_io_bus_destroy(bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) kvm->buses[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) kvm_coalesced_mmio_free(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) kvm_arch_flush_shadow_all(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) kvm_arch_destroy_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) kvm_destroy_devices(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) cleanup_srcu_struct(&kvm->irq_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) cleanup_srcu_struct(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) kvm_arch_free_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) preempt_notifier_dec();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) hardware_disable_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) mmdrop(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) module_put(kvm_chardev_ops.owner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) void kvm_get_kvm(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) refcount_inc(&kvm->users_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) EXPORT_SYMBOL_GPL(kvm_get_kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) void kvm_put_kvm(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) if (refcount_dec_and_test(&kvm->users_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) kvm_destroy_vm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) EXPORT_SYMBOL_GPL(kvm_put_kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * Used to put a reference that was taken on behalf of an object associated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * with a user-visible file descriptor, e.g. a vcpu or device, if installation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * of the new file descriptor fails and the reference cannot be transferred to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * its final owner. In such cases, the caller is still actively using @kvm and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * will fail miserably if the refcount unexpectedly hits zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) void kvm_put_kvm_no_destroy(struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) WARN_ON(refcount_dec_and_test(&kvm->users_count));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static int kvm_vm_release(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) struct kvm *kvm = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) kvm_irqfd_release(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) kvm_put_kvm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * Allocation size is twice as large as the actual dirty bitmap size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * See kvm_vm_ioctl_get_dirty_log() why this is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (!memslot->dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) * Delete a memslot by decrementing the number of used slots and shifting all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * other entries in the array forward one spot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) static inline void kvm_memslot_delete(struct kvm_memslots *slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) struct kvm_memory_slot *memslot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) struct kvm_memory_slot *mslots = slots->memslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) if (WARN_ON(slots->id_to_index[memslot->id] == -1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) slots->used_slots--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (atomic_read(&slots->lru_slot) >= slots->used_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) atomic_set(&slots->lru_slot, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) for (i = slots->id_to_index[memslot->id]; i < slots->used_slots; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) mslots[i] = mslots[i + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) slots->id_to_index[mslots[i].id] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) mslots[i] = *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) slots->id_to_index[memslot->id] = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * "Insert" a new memslot by incrementing the number of used slots. Returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * the new slot's initial index into the memslots array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) static inline int kvm_memslot_insert_back(struct kvm_memslots *slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return slots->used_slots++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * Move a changed memslot backwards in the array by shifting existing slots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) * with a higher GFN toward the front of the array. Note, the changed memslot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * itself is not preserved in the array, i.e. not swapped at this time, only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * its new index into the array is tracked. Returns the changed memslot's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * current index into the memslots array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) static inline int kvm_memslot_move_backward(struct kvm_memslots *slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) struct kvm_memory_slot *memslot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) struct kvm_memory_slot *mslots = slots->memslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (WARN_ON_ONCE(slots->id_to_index[memslot->id] == -1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) WARN_ON_ONCE(!slots->used_slots))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * Move the target memslot backward in the array by shifting existing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * memslots with a higher GFN (than the target memslot) towards the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * front of the array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) for (i = slots->id_to_index[memslot->id]; i < slots->used_slots - 1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) if (memslot->base_gfn > mslots[i + 1].base_gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /* Shift the next memslot forward one and update its index. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) mslots[i] = mslots[i + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) slots->id_to_index[mslots[i].id] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * Move a changed memslot forwards in the array by shifting existing slots with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * a lower GFN toward the back of the array. Note, the changed memslot itself
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * is not preserved in the array, i.e. not swapped at this time, only its new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * index into the array is tracked. Returns the changed memslot's final index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * into the memslots array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) static inline int kvm_memslot_move_forward(struct kvm_memslots *slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) struct kvm_memory_slot *memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) int start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) struct kvm_memory_slot *mslots = slots->memslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) for (i = start; i > 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) if (memslot->base_gfn < mslots[i - 1].base_gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) /* Shift the next memslot back one and update its index. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) mslots[i] = mslots[i - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) slots->id_to_index[mslots[i].id] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * Re-sort memslots based on their GFN to account for an added, deleted, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * moved memslot. Sorting memslots by GFN allows using a binary search during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * memslot lookup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! I.e. the entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) * at memslots[0] has the highest GFN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) * The sorting algorithm takes advantage of having initially sorted memslots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) * and knowing the position of the changed memslot. Sorting is also optimized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * by not swapping the updated memslot and instead only shifting other memslots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * and tracking the new index for the update memslot. Only once its final
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * index is known is the updated memslot copied into its position in the array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * - When deleting a memslot, the deleted memslot simply needs to be moved to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * the end of the array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * - When creating a memslot, the algorithm "inserts" the new memslot at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * end of the array and then it forward to its correct location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * - When moving a memslot, the algorithm first moves the updated memslot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * backward to handle the scenario where the memslot's GFN was changed to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * lower value. update_memslots() then falls through and runs the same flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * as creating a memslot to move the memslot forward to handle the scenario
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * where its GFN was changed to a higher value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * Note, slots are sorted from highest->lowest instead of lowest->highest for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * historical reasons. Originally, invalid memslots where denoted by having
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * GFN=0, thus sorting from highest->lowest naturally sorted invalid memslots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * to the end of the array. The current algorithm uses dedicated logic to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * delete a memslot and thus does not rely on invalid memslots having GFN=0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * The other historical motiviation for highest->lowest was to improve the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * performance of memslot lookup. KVM originally used a linear search starting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * at memslots[0]. On x86, the largest memslot usually has one of the highest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * if not *the* highest, GFN, as the bulk of the guest's RAM is located in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * single memslot above the 4gb boundary. As the largest memslot is also the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * most likely to be referenced, sorting it to the front of the array was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * advantageous. The current binary search starts from the middle of the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * and uses an LRU pointer to improve performance for all memslots and GFNs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) static void update_memslots(struct kvm_memslots *slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) struct kvm_memory_slot *memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) enum kvm_mr_change change)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (change == KVM_MR_DELETE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) kvm_memslot_delete(slots, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (change == KVM_MR_CREATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) i = kvm_memslot_insert_back(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) i = kvm_memslot_move_backward(slots, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) i = kvm_memslot_move_forward(slots, memslot, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * Copy the memslot to its new position in memslots and update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * its index accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) slots->memslots[i] = *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) slots->id_to_index[memslot->id] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) #ifdef __KVM_HAVE_READONLY_MEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) valid_flags |= KVM_MEM_READONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (mem->flags & ~valid_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) int as_id, struct kvm_memslots *slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) u64 gen = old_memslots->generation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) rcu_assign_pointer(kvm->memslots[as_id], slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) synchronize_srcu_expedited(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * Increment the new memslot generation a second time, dropping the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * update in-progress flag and incrementing the generation based on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * the number of address spaces. This provides a unique and easily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) * identifiable generation number while the memslots are in flux.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) * Generations must be unique even across address spaces. We do not need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) * a global counter for that, instead the generation space is evenly split
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) * across address spaces. For example, with two address spaces, address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) * space 0 will use generations 0, 2, 4, ... while address space 1 will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * use generations 1, 3, 5, ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) gen += KVM_ADDRESS_SPACE_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) kvm_arch_memslots_updated(kvm, gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) slots->generation = gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) return old_memslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) * Note, at a minimum, the current number of used slots must be allocated, even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * when deleting a memslot, as we need a complete duplicate of the memslots for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * use when invalidating a memslot prior to deleting/moving the memslot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) enum kvm_mr_change change)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) size_t old_size, new_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) old_size = sizeof(struct kvm_memslots) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) (sizeof(struct kvm_memory_slot) * old->used_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) if (change == KVM_MR_CREATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) new_size = old_size + sizeof(struct kvm_memory_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) new_size = old_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) slots = kvzalloc(new_size, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (likely(slots))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) memcpy(slots, old, old_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) return slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) static int kvm_set_memslot(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) const struct kvm_userspace_memory_region *mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) struct kvm_memory_slot *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) struct kvm_memory_slot *new, int as_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) enum kvm_mr_change change)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct kvm_memory_slot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) slots = kvm_dup_memslots(__kvm_memslots(kvm, as_id), change);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (!slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * Note, the INVALID flag needs to be in the appropriate entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * in the freshly allocated memslots, not in @old or @new.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) slot = id_to_memslot(slots, old->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) slot->flags |= KVM_MEMSLOT_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * We can re-use the old memslots, the only difference from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) * newly installed memslots is the invalid flag, which will get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * dropped by update_memslots anyway. We'll also revert to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) * old memslots if preparing the new memory region fails.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) slots = install_new_memslots(kvm, as_id, slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /* From this point no new shadow pages pointing to a deleted,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * or moved, memslot will be created.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * validation of sp->gfn happens in:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * - kvm_is_visible_gfn (mmu_check_root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) kvm_arch_flush_shadow_memslot(kvm, slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) r = kvm_arch_prepare_memory_region(kvm, new, mem, change);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) goto out_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) update_memslots(slots, new, change);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) slots = install_new_memslots(kvm, as_id, slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) kvm_arch_commit_memory_region(kvm, mem, old, new, change);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) out_slots:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) slots = install_new_memslots(kvm, as_id, slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) static int kvm_delete_memslot(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) const struct kvm_userspace_memory_region *mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) struct kvm_memory_slot *old, int as_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) struct kvm_memory_slot new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (!old->npages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) memset(&new, 0, sizeof(new));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) new.id = old->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) * This is only for debugging purpose; it should never be referenced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) * for a removed memslot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) new.as_id = as_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) r = kvm_set_memslot(kvm, mem, old, &new, as_id, KVM_MR_DELETE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) kvm_free_memslot(kvm, old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) * Allocate some memory and give it an address in the guest physical address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) * space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) * Discontiguous memory is allowed, mostly for framebuffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) * Must be called holding kvm->slots_lock for write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) int __kvm_set_memory_region(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) const struct kvm_userspace_memory_region *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) struct kvm_memory_slot old, new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) struct kvm_memory_slot *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) enum kvm_mr_change change;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) int as_id, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) r = check_memory_region_flags(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) as_id = mem->slot >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) id = (u16)mem->slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) /* General sanity checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if ((mem->memory_size & (PAGE_SIZE - 1)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) (mem->memory_size != (unsigned long)mem->memory_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) if (mem->guest_phys_addr & (PAGE_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) /* We can read the guest memory with __xxx_user() later on. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) (mem->userspace_addr != untagged_addr(mem->userspace_addr)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) !access_ok((void __user *)(unsigned long)mem->userspace_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) mem->memory_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) * Make a full copy of the old memslot, the pointer will become stale
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) * when the memslots are re-sorted by update_memslots(), and the old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) * memslot needs to be referenced after calling update_memslots(), e.g.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) * to free its resources and for arch specific behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) tmp = id_to_memslot(__kvm_memslots(kvm, as_id), id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) if (tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) old = *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) tmp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) memset(&old, 0, sizeof(old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) old.id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) if (!mem->memory_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) return kvm_delete_memslot(kvm, mem, &old, as_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) new.as_id = as_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) new.id = id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) new.npages = mem->memory_size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) new.flags = mem->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) new.userspace_addr = mem->userspace_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) if (new.npages > KVM_MEM_MAX_NR_PAGES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) if (!old.npages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) change = KVM_MR_CREATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) new.dirty_bitmap = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) memset(&new.arch, 0, sizeof(new.arch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) } else { /* Modify an existing slot. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if ((new.userspace_addr != old.userspace_addr) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) (new.npages != old.npages) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) ((new.flags ^ old.flags) & KVM_MEM_READONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) if (new.base_gfn != old.base_gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) change = KVM_MR_MOVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) else if (new.flags != old.flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) change = KVM_MR_FLAGS_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) else /* Nothing to change. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) /* Copy dirty_bitmap and arch from the current memslot. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) new.dirty_bitmap = old.dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) memcpy(&new.arch, &old.arch, sizeof(new.arch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) /* Check for overlaps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) kvm_for_each_memslot(tmp, __kvm_memslots(kvm, as_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) if (tmp->id == id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (!((new.base_gfn + new.npages <= tmp->base_gfn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) (new.base_gfn >= tmp->base_gfn + tmp->npages)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) /* Allocate/free page dirty bitmap as needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) new.dirty_bitmap = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) else if (!new.dirty_bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) r = kvm_alloc_dirty_bitmap(&new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) if (kvm_dirty_log_manual_protect_and_init_set(kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) bitmap_set(new.dirty_bitmap, 0, new.npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) r = kvm_set_memslot(kvm, mem, &old, &new, as_id, change);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) goto out_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) if (old.dirty_bitmap && !new.dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) kvm_destroy_dirty_bitmap(&old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) out_bitmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) if (new.dirty_bitmap && !old.dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) kvm_destroy_dirty_bitmap(&new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) int kvm_set_memory_region(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) const struct kvm_userspace_memory_region *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) r = __kvm_set_memory_region(kvm, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) EXPORT_SYMBOL_GPL(kvm_set_memory_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) struct kvm_userspace_memory_region *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return kvm_set_memory_region(kvm, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) #ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) * kvm_get_dirty_log - get a snapshot of dirty pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) * @kvm: pointer to kvm instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) * @log: slot id and address to which we copy the log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) * @is_dirty: set to '1' if any dirty pages were found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) * @memslot: set to the associated memslot, always valid on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) int *is_dirty, struct kvm_memory_slot **memslot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) int i, as_id, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) unsigned long n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) unsigned long any = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) *memslot = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) *is_dirty = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) as_id = log->slot >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) id = (u16)log->slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) slots = __kvm_memslots(kvm, as_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) *memslot = id_to_memslot(slots, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (!(*memslot) || !(*memslot)->dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) kvm_arch_sync_dirty_log(kvm, *memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) n = kvm_dirty_bitmap_bytes(*memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) for (i = 0; !any && i < n/sizeof(long); ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) any = (*memslot)->dirty_bitmap[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) if (any)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) *is_dirty = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) #else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) * kvm_get_dirty_log_protect - get a snapshot of dirty pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) * and reenable dirty page tracking for the corresponding pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * @kvm: pointer to kvm instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) * @log: slot id and address to which we copy the log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) * We need to keep it in mind that VCPU threads can write to the bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) * concurrently. So, to avoid losing track of dirty pages we keep the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) * following order:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) * 1. Take a snapshot of the bit and clear it if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) * 2. Write protect the corresponding page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) * 3. Copy the snapshot to the userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) * 4. Upon return caller flushes TLB's if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) * Between 2 and 4, the guest may write to the page using the remaining TLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) * entry. This is not a problem because the page is reported dirty using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) * the snapshot taken before and step 4 ensures that writes done after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) * exiting to userspace will be logged for the next call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) struct kvm_memory_slot *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) int i, as_id, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) unsigned long n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) unsigned long *dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) unsigned long *dirty_bitmap_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) bool flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) as_id = log->slot >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) id = (u16)log->slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) slots = __kvm_memslots(kvm, as_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) memslot = id_to_memslot(slots, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) if (!memslot || !memslot->dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) dirty_bitmap = memslot->dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) kvm_arch_sync_dirty_log(kvm, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) n = kvm_dirty_bitmap_bytes(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) flush = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (kvm->manual_dirty_log_protect) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) * Unlike kvm_get_dirty_log, we always return false in *flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) * because no flush is needed until KVM_CLEAR_DIRTY_LOG. There
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) * is some code duplication between this function and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) * kvm_get_dirty_log, but hopefully all architecture
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) * can be eliminated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) dirty_bitmap_buffer = dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) memset(dirty_bitmap_buffer, 0, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) for (i = 0; i < n / sizeof(long); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) gfn_t offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) if (!dirty_bitmap[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) flush = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) mask = xchg(&dirty_bitmap[i], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) dirty_bitmap_buffer[i] = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) offset = i * BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) offset, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) * @kvm: kvm instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) * @log: slot id and address to which we copy the log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * Steps 1-4 below provide general overview of dirty page logging. See
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * kvm_get_dirty_log_protect() function description for additional details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * always flush the TLB (step 4) even if previous step failed and the dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) * does not preclude user space subsequent dirty log read. Flushing TLB ensures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) * writes will be marked dirty for next log read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) * 1. Take a snapshot of the bit and clear it if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) * 2. Write protect the corresponding page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) * 3. Copy the snapshot to the userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) * 4. Flush TLB's if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) struct kvm_dirty_log *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) r = kvm_get_dirty_log_protect(kvm, log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) * and reenable dirty page tracking for the corresponding pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) * @kvm: pointer to kvm instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * @log: slot id and address from which to fetch the bitmap of dirty pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) static int kvm_clear_dirty_log_protect(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) struct kvm_clear_dirty_log *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) struct kvm_memslots *slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) struct kvm_memory_slot *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) int as_id, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) gfn_t offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) unsigned long i, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) unsigned long *dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) unsigned long *dirty_bitmap_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) bool flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) as_id = log->slot >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) id = (u16)log->slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) if (log->first_page & 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) slots = __kvm_memslots(kvm, as_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) memslot = id_to_memslot(slots, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) if (!memslot || !memslot->dirty_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) dirty_bitmap = memslot->dirty_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) if (log->first_page > memslot->npages ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) log->num_pages > memslot->npages - log->first_page ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) kvm_arch_sync_dirty_log(kvm, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) flush = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) spin_lock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) for (offset = log->first_page, i = offset / BITS_PER_LONG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) i++, offset += BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) unsigned long mask = *dirty_bitmap_buffer++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) if (!mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) mask &= atomic_long_fetch_andnot(mask, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) * mask contains the bits that really have been cleared. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) * never includes any bits beyond the length of the memslot (if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) * the length is not aligned to 64 pages), therefore it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) * a problem if userspace sets them in log->dirty_bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) if (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) flush = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) offset, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) spin_unlock(&kvm->mmu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) struct kvm_clear_dirty_log *log)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) mutex_lock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) r = kvm_clear_dirty_log_protect(kvm, log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) mutex_unlock(&kvm->slots_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) return __gfn_to_memslot(kvm_memslots(kvm), gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) EXPORT_SYMBOL_GPL(gfn_to_memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) return kvm_is_visible_memslot(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) return kvm_is_visible_memslot(memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) unsigned long addr, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) size = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (kvm_is_error_hva(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) return PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) mmap_read_lock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) vma = find_vma(current->mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) if (!vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) size = vma_kernel_pagesize(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) mmap_read_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) static bool memslot_is_readonly(struct kvm_memory_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) return slot->flags & KVM_MEM_READONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) gfn_t *nr_pages, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) return KVM_HVA_ERR_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) if (memslot_is_readonly(slot) && write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) return KVM_HVA_ERR_RO_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) if (nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) *nr_pages = slot->npages - (gfn - slot->base_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) return __gfn_to_hva_memslot(slot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) gfn_t *nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) return __gfn_to_hva_many(slot, gfn, nr_pages, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return gfn_to_hva_many(slot, gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) EXPORT_SYMBOL_GPL(gfn_to_hva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) * Return the hva of a @gfn and the R/W attribute if possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * @slot: the kvm_memory_slot which contains @gfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * @gfn: the gfn to be translated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * @writable: used to return the read/write attribute of the @slot if the hva
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) * is valid and @writable is not NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) gfn_t gfn, bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) if (!kvm_is_error_hva(hva) && writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) *writable = !memslot_is_readonly(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) return hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) return gfn_to_hva_memslot_prot(slot, gfn, writable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) return gfn_to_hva_memslot_prot(slot, gfn, writable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) static inline int check_user_page_hwpoison(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) rc = get_user_pages(addr, 1, flags, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) return rc == -EHWPOISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) * The fast path to get the writable pfn which will be stored in @pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) * true indicates success, otherwise false is returned. It's also the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) * only part that runs if we can in atomic context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) bool *writable, kvm_pfn_t *pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) struct page *page[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) * Fast pin a writable pfn only if it is a write fault request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) * or the caller allows to map a writable pfn for a read fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) if (!(write_fault || writable))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) if (get_user_page_fast_only(addr, FOLL_WRITE, page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) *pfn = page_to_pfn(page[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) if (writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) *writable = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) * The slow path to get the pfn of the specified host virtual address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * 1 indicates success, -errno is returned if error is detected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) bool *writable, kvm_pfn_t *pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) unsigned int flags = FOLL_HWPOISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) int npages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) if (writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) *writable = write_fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) if (write_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) flags |= FOLL_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) if (async)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) flags |= FOLL_NOWAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) npages = get_user_pages_unlocked(addr, 1, &page, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (npages != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) /* map read fault as writable if possible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) if (unlikely(!write_fault) && writable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) struct page *wpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) if (get_user_page_fast_only(addr, FOLL_WRITE, &wpage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) *writable = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) page = wpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) *pfn = page_to_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) return npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) if (unlikely(!(vma->vm_flags & VM_READ)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) static int kvm_try_get_pfn(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) if (kvm_is_reserved_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) return get_page_unless_zero(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) static int hva_to_pfn_remapped(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) unsigned long addr, bool *async,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) bool write_fault, bool *writable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) kvm_pfn_t *p_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) kvm_pfn_t pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) * not call the fault handler, so do it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) bool unlocked = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) r = fixup_user_fault(current->mm, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) (write_fault ? FAULT_FLAG_WRITE : 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) &unlocked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) if (unlocked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) r = follow_pte(vma->vm_mm, addr, &ptep, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (write_fault && !pte_write(*ptep)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) pfn = KVM_PFN_ERR_RO_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) if (writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) *writable = pte_write(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) pfn = pte_pfn(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) * Get a reference here because callers of *hva_to_pfn* and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) * returned pfn. This is only needed if the VMA has VM_MIXEDMAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) * simply do nothing for reserved pfns.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) * Whoever called remap_pfn_range is also going to call e.g.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) * unmap_mapping_range before the underlying pages are freed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) * causing a call to our MMU notifier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) * Certain IO or PFNMAP mappings can be backed with valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) * struct pages, but be allocated without refcounting e.g.,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) * tail pages of non-compound higher order allocations, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) * would then underflow the refcount when the caller does the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) * required put_page. Don't allow those pages here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) if (!kvm_try_get_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) pte_unmap_unlock(ptep, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) *p_pfn = pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) * Pin guest page in memory and return its pfn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) * @addr: host virtual address which maps memory to the guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) * @atomic: whether this function can sleep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) * @async: whether this function need to wait IO complete if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) * host page is not in the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) * @write_fault: whether we should get a writable host page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) * @writable: whether it allows to map a writable host page for !@write_fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) * The function will map a writable host page for these two cases:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) * 1): @write_fault = true
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) * 2): @write_fault = false && @writable, @writable will tell the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) * whether the mapping is writable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) bool write_fault, bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) kvm_pfn_t pfn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) int npages, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) /* we can do it either atomically or asynchronously, not both */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) BUG_ON(atomic && async);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) return pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) if (atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) return KVM_PFN_ERR_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) if (npages == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) return pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) mmap_read_lock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (npages == -EHWPOISON ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) (!async && check_user_page_hwpoison(addr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) pfn = KVM_PFN_ERR_HWPOISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) vma = find_vma_intersection(current->mm, addr, addr + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) if (vma == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) pfn = KVM_PFN_ERR_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) if (r == -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) pfn = KVM_PFN_ERR_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) if (async && vma_is_valid(vma, write_fault))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) *async = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) pfn = KVM_PFN_ERR_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) mmap_read_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) return pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) bool atomic, bool *async, bool write_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) if (addr == KVM_HVA_ERR_RO_BAD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) if (writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) *writable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) return KVM_PFN_ERR_RO_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) if (kvm_is_error_hva(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) if (writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) *writable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) return KVM_PFN_NOSLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) /* Do not map writable pfn in the readonly memslot. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) if (writable && memslot_is_readonly(slot)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) *writable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) writable = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) return hva_to_pfn(addr, atomic, async, write_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) writable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) bool *writable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) write_fault, writable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) EXPORT_SYMBOL_GPL(gfn_to_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) struct page **pages, int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) gfn_t entry = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) addr = gfn_to_hva_many(slot, gfn, &entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) if (kvm_is_error_hva(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) if (entry < nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) if (is_error_noslot_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) return KVM_ERR_PTR_BAD_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) if (kvm_is_reserved_pfn(pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) return KVM_ERR_PTR_BAD_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) return pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) kvm_pfn_t pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) pfn = gfn_to_pfn(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) return kvm_pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) EXPORT_SYMBOL_GPL(gfn_to_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) void kvm_release_pfn(kvm_pfn_t pfn, bool dirty, struct gfn_to_pfn_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) if (pfn == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) if (cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) cache->pfn = cache->gfn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) if (dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) kvm_release_pfn_dirty(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) kvm_release_pfn_clean(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) static void kvm_cache_gfn_to_pfn(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) struct gfn_to_pfn_cache *cache, u64 gen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) kvm_release_pfn(cache->pfn, cache->dirty, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) cache->pfn = gfn_to_pfn_memslot(slot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) cache->gfn = gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) cache->dirty = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) cache->generation = gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) static int __kvm_map_gfn(struct kvm_memslots *slots, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) struct kvm_host_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) struct gfn_to_pfn_cache *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) bool atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) kvm_pfn_t pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) void *hva = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) struct page *page = KVM_UNMAPPED_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) struct kvm_memory_slot *slot = __gfn_to_memslot(slots, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) u64 gen = slots->generation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) if (!map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) if (cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) if (!cache->pfn || cache->gfn != gfn ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) cache->generation != gen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) if (atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) kvm_cache_gfn_to_pfn(slot, gfn, cache, gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) pfn = cache->pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) if (atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) pfn = gfn_to_pfn_memslot(slot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) if (is_error_noslot_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) if (pfn_valid(pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) page = pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) if (atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) hva = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) hva = kmap(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) #ifdef CONFIG_HAS_IOMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) } else if (!atomic) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) hva = memremap(pfn_to_hpa(pfn), PAGE_SIZE, MEMREMAP_WB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) if (!hva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) map->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) map->hva = hva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) map->pfn = pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) map->gfn = gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) int kvm_map_gfn(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) struct gfn_to_pfn_cache *cache, bool atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) return __kvm_map_gfn(kvm_memslots(vcpu->kvm), gfn, map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) cache, atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) EXPORT_SYMBOL_GPL(kvm_map_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) return __kvm_map_gfn(kvm_vcpu_memslots(vcpu), gfn, map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) NULL, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) EXPORT_SYMBOL_GPL(kvm_vcpu_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) static void __kvm_unmap_gfn(struct kvm_memory_slot *memslot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) struct kvm_host_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) struct gfn_to_pfn_cache *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) bool dirty, bool atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) if (!map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (!map->hva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) if (map->page != KVM_UNMAPPED_PAGE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) if (atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) kunmap_atomic(map->hva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) kunmap(map->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) #ifdef CONFIG_HAS_IOMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) else if (!atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) memunmap(map->hva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) WARN_ONCE(1, "Unexpected unmapping in atomic context");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) if (dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) mark_page_dirty_in_slot(memslot, map->gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) if (cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) cache->dirty |= dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) kvm_release_pfn(map->pfn, dirty, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) map->hva = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) map->page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) int kvm_unmap_gfn(struct kvm_vcpu *vcpu, struct kvm_host_map *map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) struct gfn_to_pfn_cache *cache, bool dirty, bool atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) __kvm_unmap_gfn(gfn_to_memslot(vcpu->kvm, map->gfn), map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) cache, dirty, atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) EXPORT_SYMBOL_GPL(kvm_unmap_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map, bool dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) __kvm_unmap_gfn(kvm_vcpu_gfn_to_memslot(vcpu, map->gfn), map, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) dirty, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) kvm_pfn_t pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) return kvm_pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) void kvm_release_page_clean(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) WARN_ON(is_error_page(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) kvm_release_pfn_clean(page_to_pfn(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) EXPORT_SYMBOL_GPL(kvm_release_page_clean);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) void kvm_release_pfn_clean(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) put_page(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) void kvm_release_page_dirty(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) WARN_ON(is_error_page(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) kvm_release_pfn_dirty(page_to_pfn(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) void kvm_release_pfn_dirty(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) kvm_set_pfn_dirty(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) kvm_release_pfn_clean(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) void kvm_set_pfn_dirty(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) SetPageDirty(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) void kvm_set_pfn_accessed(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) if (!kvm_is_reserved_pfn(pfn) && !kvm_is_zone_device_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) mark_page_accessed(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) void kvm_get_pfn(kvm_pfn_t pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) if (!kvm_is_reserved_pfn(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) get_page(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) EXPORT_SYMBOL_GPL(kvm_get_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) static int next_segment(unsigned long len, int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (len > PAGE_SIZE - offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) return PAGE_SIZE - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) void *data, int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) if (kvm_is_error_hva(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) r = __copy_from_user(data, (void __user *)addr + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) return __kvm_read_guest_page(slot, gfn, data, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) EXPORT_SYMBOL_GPL(kvm_read_guest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) return __kvm_read_guest_page(slot, gfn, data, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) while ((seg = next_segment(len, offset)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) len -= seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) data += seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) ++gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) EXPORT_SYMBOL_GPL(kvm_read_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) while ((seg = next_segment(len, offset)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) len -= seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) data += seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) ++gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) void *data, int offset, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) if (kvm_is_error_hva(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) void *data, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) const void *data, int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) addr = gfn_to_hva_memslot(memslot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) if (kvm_is_error_hva(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) r = __copy_to_user((void __user *)addr + offset, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) mark_page_dirty_in_slot(memslot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) const void *data, int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) return __kvm_write_guest_page(slot, gfn, data, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) EXPORT_SYMBOL_GPL(kvm_write_guest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) const void *data, int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) return __kvm_write_guest_page(slot, gfn, data, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) while ((seg = next_segment(len, offset)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) len -= seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) data += seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) ++gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) EXPORT_SYMBOL_GPL(kvm_write_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) while ((seg = next_segment(len, offset)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) len -= seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) data += seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) ++gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) gpa_t gpa, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) gfn_t start_gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) gfn_t nr_pages_avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) /* Update ghc->generation before performing any error checks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) ghc->generation = slots->generation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) if (start_gfn > end_gfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) ghc->hva = KVM_HVA_ERR_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) * If the requested region crosses two memslots, we still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) * verify that the entire region is valid here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) ghc->memslot = __gfn_to_memslot(slots, start_gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) &nr_pages_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) if (kvm_is_error_hva(ghc->hva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) /* Use the slow path for cross page reads and writes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) if (nr_pages_needed == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) ghc->hva += offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) ghc->memslot = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) ghc->gpa = gpa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) ghc->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) gpa_t gpa, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) struct kvm_memslots *slots = kvm_memslots(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) void *data, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) struct kvm_memslots *slots = kvm_memslots(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) gpa_t gpa = ghc->gpa + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) if (WARN_ON_ONCE(len + offset > ghc->len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) if (slots->generation != ghc->generation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) if (kvm_is_error_hva(ghc->hva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) if (unlikely(!ghc->memslot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) return kvm_write_guest(kvm, gpa, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) void *data, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) void *data, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) struct kvm_memslots *slots = kvm_memslots(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) gpa_t gpa = ghc->gpa + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) if (WARN_ON_ONCE(len + offset > ghc->len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) if (slots->generation != ghc->generation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) if (kvm_is_error_hva(ghc->hva))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) if (unlikely(!ghc->memslot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) return kvm_read_guest(kvm, gpa, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) r = __copy_from_user(data, (void __user *)ghc->hva + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) void *data, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) gfn_t gfn = gpa >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) int offset = offset_in_page(gpa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) while ((seg = next_segment(len, offset)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) len -= seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) ++gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) EXPORT_SYMBOL_GPL(kvm_clear_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) if (memslot && memslot->dirty_bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) unsigned long rel_gfn = gfn - memslot->base_gfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) set_bit_le(rel_gfn, memslot->dirty_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) struct kvm_memory_slot *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) memslot = gfn_to_memslot(kvm, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) mark_page_dirty_in_slot(memslot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) EXPORT_SYMBOL_GPL(mark_page_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) struct kvm_memory_slot *memslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) mark_page_dirty_in_slot(memslot, gfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) void kvm_sigset_activate(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) if (!vcpu->sigset_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) * This does a lockless modification of ->real_blocked, which is fine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) * because, only current can change ->real_blocked and all readers of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) * ->real_blocked don't care as long ->real_blocked is always a subset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) * of ->blocked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) sigprocmask(SIG_SETMASK, &vcpu->sigset, ¤t->real_blocked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) if (!vcpu->sigset_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) sigprocmask(SIG_SETMASK, ¤t->real_blocked, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) sigemptyset(¤t->real_blocked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) unsigned int old, val, grow, grow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) old = val = vcpu->halt_poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) grow_start = READ_ONCE(halt_poll_ns_grow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) grow = READ_ONCE(halt_poll_ns_grow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) if (!grow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) val *= grow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) if (val < grow_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) val = grow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) if (val > vcpu->kvm->max_halt_poll_ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) val = vcpu->kvm->max_halt_poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) vcpu->halt_poll_ns = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) unsigned int old, val, shrink, grow_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) old = val = vcpu->halt_poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) shrink = READ_ONCE(halt_poll_ns_shrink);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) grow_start = READ_ONCE(halt_poll_ns_grow_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) if (shrink == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) val /= shrink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) if (val < grow_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) vcpu->halt_poll_ns = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) int ret = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) int idx = srcu_read_lock(&vcpu->kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) if (kvm_arch_vcpu_runnable(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) kvm_make_request(KVM_REQ_UNHALT, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) if (kvm_cpu_has_pending_timer(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) srcu_read_unlock(&vcpu->kvm->srcu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) update_halt_poll_stats(struct kvm_vcpu *vcpu, u64 poll_ns, bool waited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) if (waited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) vcpu->stat.halt_poll_fail_ns += poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) vcpu->stat.halt_poll_success_ns += poll_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) * The vCPU has executed a HLT instruction with in-kernel mode enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) void kvm_vcpu_block(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) ktime_t start, cur, poll_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) bool waited = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) u64 block_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) kvm_arch_vcpu_blocking(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) start = cur = poll_end = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) ++vcpu->stat.halt_attempted_poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) * This sets KVM_REQ_UNHALT if an interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) * arrives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) if (kvm_vcpu_check_block(vcpu) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) ++vcpu->stat.halt_successful_poll;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) if (!vcpu_valid_wakeup(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) ++vcpu->stat.halt_poll_invalid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) poll_end = cur = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) } while (single_task_running() && !need_resched() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) ktime_before(cur, stop));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) prepare_to_rcuwait(&vcpu->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) if (kvm_vcpu_check_block(vcpu) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) waited = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) finish_rcuwait(&vcpu->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) cur = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) kvm_arch_vcpu_unblocking(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) update_halt_poll_stats(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) vcpu, ktime_to_ns(ktime_sub(poll_end, start)), waited);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) if (!kvm_arch_no_poll(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) if (!vcpu_valid_wakeup(vcpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) shrink_halt_poll_ns(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) } else if (vcpu->kvm->max_halt_poll_ns) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) if (block_ns <= vcpu->halt_poll_ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) /* we had a long block, shrink polling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) else if (vcpu->halt_poll_ns &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) block_ns > vcpu->kvm->max_halt_poll_ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) shrink_halt_poll_ns(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) /* we had a short halt and our poll time is too small */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) else if (vcpu->halt_poll_ns < vcpu->kvm->max_halt_poll_ns &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) block_ns < vcpu->kvm->max_halt_poll_ns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) grow_halt_poll_ns(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) vcpu->halt_poll_ns = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) kvm_arch_vcpu_block_finish(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) EXPORT_SYMBOL_GPL(kvm_vcpu_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) struct rcuwait *waitp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) waitp = kvm_arch_vcpu_get_wait(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) if (rcuwait_wake_up(waitp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) WRITE_ONCE(vcpu->ready, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) ++vcpu->stat.halt_wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) #ifndef CONFIG_S390
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) int me;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) int cpu = vcpu->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) if (kvm_vcpu_wake_up(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) me = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) if (kvm_arch_vcpu_should_kick(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) smp_send_reschedule(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) #endif /* !CONFIG_S390 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) int kvm_vcpu_yield_to(struct kvm_vcpu *target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) struct pid *pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) struct task_struct *task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) pid = rcu_dereference(target->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) if (pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) task = get_pid_task(pid, PIDTYPE_PID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) if (!task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) ret = yield_to(task, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) put_task_struct(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) * Helper that checks whether a VCPU is eligible for directed yield.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) * Most eligible candidate to yield is decided by following heuristics:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) * (a) VCPU which has not done pl-exit or cpu relax intercepted recently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) * (preempted lock holder), indicated by @in_spin_loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) * Set at the beginning and cleared at the end of interception/PLE handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) * chance last time (mostly it has become eligible now since we have probably
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) * yielded to lockholder in last iteration. This is done by toggling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) * @dy_eligible each time a VCPU checked for eligibility.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) * Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) * to preempted lock-holder could result in wrong VCPU selection and CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) * burning. Giving priority for a potential lock-holder increases lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) * progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) * Since algorithm is based on heuristics, accessing another VCPU data without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) * locking does not harm. It may result in trying to yield to same VCPU, fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) * and continue with next VCPU and so on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) bool eligible;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) eligible = !vcpu->spin_loop.in_spin_loop ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) vcpu->spin_loop.dy_eligible;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) if (vcpu->spin_loop.in_spin_loop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) return eligible;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) * Unlike kvm_arch_vcpu_runnable, this function is called outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) * a vcpu_load/vcpu_put pair. However, for most architectures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) * kvm_arch_vcpu_runnable does not require vcpu_load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) return kvm_arch_vcpu_runnable(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) if (kvm_arch_dy_runnable(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) #ifdef CONFIG_KVM_ASYNC_PF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) if (!list_empty_careful(&vcpu->async_pf.done))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) struct kvm *kvm = me->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) int yielded = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) int try = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) int pass;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) kvm_vcpu_set_in_spin_loop(me, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) * We boost the priority of a VCPU that is runnable but not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) * currently running, because it got preempted by something
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) * else and called schedule in __vcpu_run. Hopefully that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) * VCPU is holding the lock that we need and will release it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) * We approximate round-robin by starting at the last boosted VCPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) for (pass = 0; pass < 2 && !yielded && try; pass++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) kvm_for_each_vcpu(i, vcpu, kvm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) if (!pass && i <= last_boosted_vcpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) i = last_boosted_vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) } else if (pass && i > last_boosted_vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) if (!READ_ONCE(vcpu->ready))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) if (vcpu == me)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) if (rcuwait_active(&vcpu->wait) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) !vcpu_dy_runnable(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) !kvm_arch_vcpu_in_kernel(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) yielded = kvm_vcpu_yield_to(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (yielded > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) kvm->last_boosted_vcpu = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) } else if (yielded < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) try--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) if (!try)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) kvm_vcpu_set_in_spin_loop(me, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) /* Ensure vcpu is not eligible during next spinloop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) kvm_vcpu_set_dy_eligible(me, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) if (vmf->pgoff == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) page = virt_to_page(vcpu->run);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) page = virt_to_page(vcpu->arch.pio_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) #ifdef CONFIG_KVM_MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) return kvm_arch_vcpu_fault(vcpu, vmf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) vmf->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) static const struct vm_operations_struct kvm_vcpu_vm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) .fault = kvm_vcpu_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) vma->vm_ops = &kvm_vcpu_vm_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) static int kvm_vcpu_release(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) struct kvm_vcpu *vcpu = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) kvm_put_kvm(vcpu->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) static struct file_operations kvm_vcpu_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) .release = kvm_vcpu_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) .unlocked_ioctl = kvm_vcpu_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) .mmap = kvm_vcpu_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) KVM_COMPAT(kvm_vcpu_compat_ioctl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) * Allocates an inode for the vcpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) static int create_vcpu_fd(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) char name[8 + 1 + ITOA_MAX_LEN + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) struct dentry *debugfs_dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) char dir_name[ITOA_MAX_LEN * 2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) if (!debugfs_initialized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) debugfs_dentry = debugfs_create_dir(dir_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) vcpu->kvm->debugfs_dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) * Creates some virtual cpus. Good luck creating more than one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) if (id >= KVM_MAX_VCPU_ID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) if (kvm->created_vcpus == KVM_MAX_VCPUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) kvm->created_vcpus++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) r = kvm_arch_vcpu_precreate(kvm, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) goto vcpu_decrement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) if (!vcpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) goto vcpu_decrement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) page = alloc_page(GFP_KERNEL | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) goto vcpu_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) vcpu->run = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) kvm_vcpu_init(vcpu, kvm, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) r = kvm_arch_vcpu_create(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) goto vcpu_free_run_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) if (kvm_get_vcpu_by_id(kvm, id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) r = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) goto unlock_vcpu_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) BUG_ON(kvm->vcpus[vcpu->vcpu_idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) /* Now it's all set up, let userspace reach it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) kvm_get_kvm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) r = create_vcpu_fd(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) if (r < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) kvm_put_kvm_no_destroy(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) goto unlock_vcpu_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) kvm->vcpus[vcpu->vcpu_idx] = vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * Pairs with smp_rmb() in kvm_get_vcpu. Write kvm->vcpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) * before kvm->online_vcpu's incremented value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) atomic_inc(&kvm->online_vcpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) kvm_arch_vcpu_postcreate(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) kvm_create_vcpu_debugfs(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) unlock_vcpu_destroy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) kvm_arch_vcpu_destroy(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) vcpu_free_run_page:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) free_page((unsigned long)vcpu->run);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) vcpu_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) kmem_cache_free(kvm_vcpu_cache, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) vcpu_decrement:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) kvm->created_vcpus--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) if (sigset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) vcpu->sigset_active = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) vcpu->sigset = *sigset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) vcpu->sigset_active = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) static long kvm_vcpu_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) unsigned int ioctl, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) struct kvm_vcpu *vcpu = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) void __user *argp = (void __user *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) struct kvm_fpu *fpu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) struct kvm_sregs *kvm_sregs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) if (vcpu->kvm->mm != current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) * Some architectures have vcpu ioctls that are asynchronous to vcpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) * execution; mutex_lock() would break them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) if (r != -ENOIOCTLCMD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) if (mutex_lock_killable(&vcpu->mutex))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) case KVM_RUN: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) struct pid *oldpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) if (arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) oldpid = rcu_access_pointer(vcpu->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) if (unlikely(oldpid != task_pid(current))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) /* The thread running this VCPU changed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) struct pid *newpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) r = kvm_arch_vcpu_run_pid_change(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) newpid = get_task_pid(current, PIDTYPE_PID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) rcu_assign_pointer(vcpu->pid, newpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) if (oldpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) put_pid(oldpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) r = kvm_arch_vcpu_ioctl_run(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) case KVM_GET_REGS: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) struct kvm_regs *kvm_regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) if (!kvm_regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) goto out_free1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) goto out_free1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) out_free1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) kfree(kvm_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) case KVM_SET_REGS: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) struct kvm_regs *kvm_regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) if (IS_ERR(kvm_regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) r = PTR_ERR(kvm_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) kfree(kvm_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) case KVM_GET_SREGS: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) kvm_sregs = kzalloc(sizeof(struct kvm_sregs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) if (!kvm_sregs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) case KVM_SET_SREGS: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) if (IS_ERR(kvm_sregs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) r = PTR_ERR(kvm_sregs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) kvm_sregs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) case KVM_GET_MP_STATE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) struct kvm_mp_state mp_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) case KVM_SET_MP_STATE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) struct kvm_mp_state mp_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) case KVM_TRANSLATE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) struct kvm_translation tr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) if (copy_from_user(&tr, argp, sizeof(tr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) if (copy_to_user(argp, &tr, sizeof(tr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) case KVM_SET_GUEST_DEBUG: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) struct kvm_guest_debug dbg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) if (copy_from_user(&dbg, argp, sizeof(dbg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) case KVM_SET_SIGNAL_MASK: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) struct kvm_signal_mask __user *sigmask_arg = argp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) struct kvm_signal_mask kvm_sigmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) sigset_t sigset, *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) p = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) if (argp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) if (copy_from_user(&kvm_sigmask, argp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) sizeof(kvm_sigmask)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) if (kvm_sigmask.len != sizeof(sigset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) if (copy_from_user(&sigset, sigmask_arg->sigset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) sizeof(sigset)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) p = &sigset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) case KVM_GET_FPU: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) if (!fpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) case KVM_SET_FPU: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) fpu = memdup_user(argp, sizeof(*fpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) if (IS_ERR(fpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) r = PTR_ERR(fpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) fpu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) mutex_unlock(&vcpu->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) kfree(fpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) kfree(kvm_sregs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) #ifdef CONFIG_KVM_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) static long kvm_vcpu_compat_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) unsigned int ioctl, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) struct kvm_vcpu *vcpu = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) void __user *argp = compat_ptr(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) if (vcpu->kvm->mm != current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) case KVM_SET_SIGNAL_MASK: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) struct kvm_signal_mask __user *sigmask_arg = argp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) struct kvm_signal_mask kvm_sigmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) sigset_t sigset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) if (argp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) if (copy_from_user(&kvm_sigmask, argp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) sizeof(kvm_sigmask)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) if (kvm_sigmask.len != sizeof(compat_sigset_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) if (get_compat_sigset(&sigset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) (compat_sigset_t __user *)sigmask_arg->sigset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) r = kvm_vcpu_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) struct kvm_device *dev = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) if (dev->ops->mmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) return dev->ops->mmap(dev, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) static int kvm_device_ioctl_attr(struct kvm_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) int (*accessor)(struct kvm_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) struct kvm_device_attr *attr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) struct kvm_device_attr attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) if (!accessor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) return accessor(dev, &attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) struct kvm_device *dev = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) if (dev->kvm->mm != current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) case KVM_SET_DEVICE_ATTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) case KVM_GET_DEVICE_ATTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) case KVM_HAS_DEVICE_ATTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) if (dev->ops->ioctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) return dev->ops->ioctl(dev, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) return -ENOTTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) static int kvm_device_release(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) struct kvm_device *dev = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) struct kvm *kvm = dev->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) if (dev->ops->release) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) list_del(&dev->vm_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) dev->ops->release(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) kvm_put_kvm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) static const struct file_operations kvm_device_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) .unlocked_ioctl = kvm_device_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) .release = kvm_device_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) KVM_COMPAT(kvm_device_ioctl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) .mmap = kvm_device_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) struct kvm_device *kvm_device_from_filp(struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) if (filp->f_op != &kvm_device_fops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) return filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) #ifdef CONFIG_KVM_MPIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) [KVM_DEV_TYPE_FSL_MPIC_20] = &kvm_mpic_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) [KVM_DEV_TYPE_FSL_MPIC_42] = &kvm_mpic_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) if (type >= ARRAY_SIZE(kvm_device_ops_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) if (kvm_device_ops_table[type] != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) kvm_device_ops_table[type] = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) void kvm_unregister_device_ops(u32 type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) if (kvm_device_ops_table[type] != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) kvm_device_ops_table[type] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) static int kvm_ioctl_create_device(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) struct kvm_create_device *cd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) const struct kvm_device_ops *ops = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) struct kvm_device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) ops = kvm_device_ops_table[type];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) if (ops == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) if (test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) dev->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) dev->kvm = kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) ret = ops->create(dev, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) kfree(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) list_add(&dev->vm_node, &kvm->devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) if (ops->init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) ops->init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) kvm_get_kvm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) kvm_put_kvm_no_destroy(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) list_del(&dev->vm_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) ops->destroy(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) cd->fd = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) switch (arg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) case KVM_CAP_USER_MEMORY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) case KVM_CAP_INTERNAL_ERROR_DATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) #ifdef CONFIG_HAVE_KVM_MSI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) case KVM_CAP_SIGNAL_MSI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) #ifdef CONFIG_HAVE_KVM_IRQFD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) case KVM_CAP_IRQFD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) case KVM_CAP_IRQFD_RESAMPLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) case KVM_CAP_IOEVENTFD_ANY_LENGTH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) case KVM_CAP_CHECK_EXTENSION_VM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) case KVM_CAP_ENABLE_CAP_VM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) case KVM_CAP_HALT_POLL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) #ifdef CONFIG_KVM_MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) case KVM_CAP_COALESCED_MMIO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) return KVM_COALESCED_MMIO_PAGE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) case KVM_CAP_COALESCED_PIO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) return KVM_DIRTY_LOG_MANUAL_CAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) case KVM_CAP_IRQ_ROUTING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) return KVM_MAX_IRQ_ROUTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) #if KVM_ADDRESS_SPACE_NUM > 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) case KVM_CAP_MULTI_ADDRESS_SPACE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) return KVM_ADDRESS_SPACE_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) case KVM_CAP_NR_MEMSLOTS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) return KVM_USER_MEM_SLOTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) return kvm_vm_ioctl_check_extension(kvm, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) struct kvm_enable_cap *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) struct kvm_enable_cap *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) switch (cap->cap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) if (cap->flags || (cap->args[0] & ~allowed_options))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) kvm->manual_dirty_log_protect = cap->args[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) case KVM_CAP_HALT_POLL: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) if (cap->flags || cap->args[0] != (unsigned int)cap->args[0])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) kvm->max_halt_poll_ns = cap->args[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) return kvm_vm_ioctl_enable_cap(kvm, cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) static long kvm_vm_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) unsigned int ioctl, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) struct kvm *kvm = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) void __user *argp = (void __user *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) if (kvm->mm != current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) case KVM_CREATE_VCPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) r = kvm_vm_ioctl_create_vcpu(kvm, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) case KVM_ENABLE_CAP: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) struct kvm_enable_cap cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) if (copy_from_user(&cap, argp, sizeof(cap)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) case KVM_SET_USER_MEMORY_REGION: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) struct kvm_userspace_memory_region kvm_userspace_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) if (copy_from_user(&kvm_userspace_mem, argp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) sizeof(kvm_userspace_mem)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) case KVM_GET_DIRTY_LOG: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) struct kvm_dirty_log log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) if (copy_from_user(&log, argp, sizeof(log)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) case KVM_CLEAR_DIRTY_LOG: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) struct kvm_clear_dirty_log log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) if (copy_from_user(&log, argp, sizeof(log)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) #ifdef CONFIG_KVM_MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) case KVM_REGISTER_COALESCED_MMIO: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) struct kvm_coalesced_mmio_zone zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) if (copy_from_user(&zone, argp, sizeof(zone)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) case KVM_UNREGISTER_COALESCED_MMIO: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) struct kvm_coalesced_mmio_zone zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) if (copy_from_user(&zone, argp, sizeof(zone)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) case KVM_IRQFD: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) struct kvm_irqfd data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) if (copy_from_user(&data, argp, sizeof(data)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) r = kvm_irqfd(kvm, &data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) case KVM_IOEVENTFD: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) struct kvm_ioeventfd data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) if (copy_from_user(&data, argp, sizeof(data)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) r = kvm_ioeventfd(kvm, &data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) #ifdef CONFIG_HAVE_KVM_MSI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) case KVM_SIGNAL_MSI: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) struct kvm_msi msi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) if (copy_from_user(&msi, argp, sizeof(msi)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) r = kvm_send_userspace_msi(kvm, &msi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) #ifdef __KVM_HAVE_IRQ_LINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) case KVM_IRQ_LINE_STATUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) case KVM_IRQ_LINE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) struct kvm_irq_level irq_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) ioctl == KVM_IRQ_LINE_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) if (ioctl == KVM_IRQ_LINE_STATUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) case KVM_SET_GSI_ROUTING: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) struct kvm_irq_routing routing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) struct kvm_irq_routing __user *urouting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) struct kvm_irq_routing_entry *entries = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) if (copy_from_user(&routing, argp, sizeof(routing)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) if (!kvm_arch_can_set_irq_routing(kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) if (routing.nr > KVM_MAX_IRQ_ROUTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) if (routing.flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) if (routing.nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) urouting = argp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) entries = vmemdup_user(urouting->entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) array_size(sizeof(*entries),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) routing.nr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) if (IS_ERR(entries)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) r = PTR_ERR(entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) r = kvm_set_irq_routing(kvm, entries, routing.nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) routing.flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) kvfree(entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) case KVM_CREATE_DEVICE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) struct kvm_create_device cd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) if (copy_from_user(&cd, argp, sizeof(cd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) r = kvm_ioctl_create_device(kvm, &cd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) if (copy_to_user(argp, &cd, sizeof(cd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) case KVM_CHECK_EXTENSION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) r = kvm_arch_vm_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) #ifdef CONFIG_KVM_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) struct compat_kvm_dirty_log {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) __u32 slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) __u32 padding1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) compat_uptr_t dirty_bitmap; /* one bit per page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) __u64 padding2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) struct compat_kvm_clear_dirty_log {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) __u32 slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) __u32 num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) __u64 first_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) compat_uptr_t dirty_bitmap; /* one bit per page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) __u64 padding2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) static long kvm_vm_compat_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) unsigned int ioctl, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) struct kvm *kvm = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) if (kvm->mm != current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) case KVM_CLEAR_DIRTY_LOG: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) struct compat_kvm_clear_dirty_log compat_log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) struct kvm_clear_dirty_log log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) if (copy_from_user(&compat_log, (void __user *)arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) sizeof(compat_log)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) log.slot = compat_log.slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) log.num_pages = compat_log.num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) log.first_page = compat_log.first_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) log.padding2 = compat_log.padding2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) case KVM_GET_DIRTY_LOG: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) struct compat_kvm_dirty_log compat_log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) struct kvm_dirty_log log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) if (copy_from_user(&compat_log, (void __user *)arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) sizeof(compat_log)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) log.slot = compat_log.slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) log.padding1 = compat_log.padding1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) log.padding2 = compat_log.padding2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) r = kvm_vm_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) static struct file_operations kvm_vm_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) .release = kvm_vm_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) .unlocked_ioctl = kvm_vm_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) KVM_COMPAT(kvm_vm_compat_ioctl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) static int kvm_dev_ioctl_create_vm(unsigned long type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) struct file *file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) kvm = kvm_create_vm(type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) if (IS_ERR(kvm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) return PTR_ERR(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) #ifdef CONFIG_KVM_MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) r = kvm_coalesced_mmio_init(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) goto put_kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) r = get_unused_fd_flags(O_CLOEXEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) goto put_kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) if (IS_ERR(file)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) put_unused_fd(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) r = PTR_ERR(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) goto put_kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) * Don't call kvm_put_kvm anymore at this point; file->f_op is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) * already set, with ->release() being kvm_vm_release(). In error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) * cases it will be called by the final fput(file) and will take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) * care of doing kvm_put_kvm(kvm).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) if (kvm_create_vm_debugfs(kvm, r) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) put_unused_fd(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) fput(file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) fd_install(r, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) put_kvm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) kvm_put_kvm(kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) static long kvm_dev_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) unsigned int ioctl, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) long r = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) switch (ioctl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) case KVM_GET_API_VERSION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) if (arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) r = KVM_API_VERSION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) case KVM_CREATE_VM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) r = kvm_dev_ioctl_create_vm(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) case KVM_CHECK_EXTENSION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) case KVM_GET_VCPU_MMAP_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) if (arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) r = PAGE_SIZE; /* struct kvm_run */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) r += PAGE_SIZE; /* pio data page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) #ifdef CONFIG_KVM_MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) r += PAGE_SIZE; /* coalesced mmio ring page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) case KVM_TRACE_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) case KVM_TRACE_PAUSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) case KVM_TRACE_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) r = -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) return kvm_arch_dev_ioctl(filp, ioctl, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) static struct file_operations kvm_chardev_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) .unlocked_ioctl = kvm_dev_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) KVM_COMPAT(kvm_dev_ioctl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) static struct miscdevice kvm_dev = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) KVM_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) "kvm",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) &kvm_chardev_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) static void hardware_enable_nolock(void *junk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) cpumask_set_cpu(cpu, cpus_hardware_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) r = kvm_arch_hardware_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) cpumask_clear_cpu(cpu, cpus_hardware_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) atomic_inc(&hardware_enable_failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) static int kvm_starting_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) raw_spin_lock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) if (kvm_usage_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) hardware_enable_nolock(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) raw_spin_unlock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) static void hardware_disable_nolock(void *junk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) cpumask_clear_cpu(cpu, cpus_hardware_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) kvm_arch_hardware_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) static int kvm_dying_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) raw_spin_lock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) if (kvm_usage_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) hardware_disable_nolock(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) raw_spin_unlock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) static void hardware_disable_all_nolock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) BUG_ON(!kvm_usage_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) kvm_usage_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) if (!kvm_usage_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) on_each_cpu(hardware_disable_nolock, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) static void hardware_disable_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) raw_spin_lock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) hardware_disable_all_nolock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) raw_spin_unlock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) static int hardware_enable_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) int r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) raw_spin_lock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) kvm_usage_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) if (kvm_usage_count == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) atomic_set(&hardware_enable_failed, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) on_each_cpu(hardware_enable_nolock, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) if (atomic_read(&hardware_enable_failed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) hardware_disable_all_nolock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) r = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) raw_spin_unlock(&kvm_count_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) * Some (well, at least mine) BIOSes hang on reboot if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) * in vmx root mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) * And Intel TXT required VMX off for all cpu when system shutdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) pr_info("kvm: exiting hardware virtualization\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) kvm_rebooting = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) on_each_cpu(hardware_disable_nolock, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) static struct notifier_block kvm_reboot_notifier = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) .notifier_call = kvm_reboot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) .priority = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) for (i = 0; i < bus->dev_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) struct kvm_io_device *pos = bus->range[i].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) kvm_iodevice_destructor(pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) kfree(bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) const struct kvm_io_range *r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) gpa_t addr1 = r1->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) gpa_t addr2 = r2->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) if (addr1 < addr2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) /* If r2->len == 0, match the exact address. If r2->len != 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) * accept any overlapping write. Any order is acceptable for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) * overlapping ranges, because kvm_io_bus_get_first_dev ensures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) * we process all of them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) if (r2->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) addr1 += r1->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) addr2 += r2->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) if (addr1 > addr2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) return kvm_io_bus_cmp(p1, p2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) gpa_t addr, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) struct kvm_io_range *range, key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) int off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) key = (struct kvm_io_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) range = bsearch(&key, bus->range, bus->dev_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) if (range == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) off = range - bus->range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) off--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) return off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) struct kvm_io_range *range, const void *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) if (idx < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) while (idx < bus->dev_count &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) range->len, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) /* kvm_io_bus_write - called under kvm->slots_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) int len, const void *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) struct kvm_io_bus *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) struct kvm_io_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) range = (struct kvm_io_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) r = __kvm_io_bus_write(vcpu, bus, &range, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) return r < 0 ? r : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) EXPORT_SYMBOL_GPL(kvm_io_bus_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) gpa_t addr, int len, const void *val, long cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) struct kvm_io_bus *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) struct kvm_io_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) range = (struct kvm_io_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) /* First try the device referenced by cookie. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) if ((cookie >= 0) && (cookie < bus->dev_count) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) return cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) * cookie contained garbage; fall back to search and return the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) * correct cookie value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) return __kvm_io_bus_write(vcpu, bus, &range, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) struct kvm_io_range *range, void *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) if (idx < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) while (idx < bus->dev_count &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) range->len, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) /* kvm_io_bus_read - called under kvm->slots_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) int len, void *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) struct kvm_io_bus *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) struct kvm_io_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) range = (struct kvm_io_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) r = __kvm_io_bus_read(vcpu, bus, &range, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) return r < 0 ? r : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) /* Caller must hold slots_lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) int len, struct kvm_io_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) struct kvm_io_bus *new_bus, *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) struct kvm_io_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) bus = kvm_get_bus(kvm, bus_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) /* exclude ioeventfd which is limited by maximum fd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) if (!new_bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) range = (struct kvm_io_range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) .dev = dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) for (i = 0; i < bus->dev_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) new_bus->dev_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) new_bus->range[i] = range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) memcpy(new_bus->range + i + 1, bus->range + i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) (bus->dev_count - i) * sizeof(struct kvm_io_range));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) synchronize_srcu_expedited(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) kfree(bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) /* Caller must hold slots_lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) struct kvm_io_device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) struct kvm_io_bus *new_bus, *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) bus = kvm_get_bus(kvm, bus_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) for (i = 0; i < bus->dev_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) if (bus->range[i].dev == dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) if (i == bus->dev_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) if (new_bus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) memcpy(new_bus, bus, struct_size(bus, range, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) new_bus->dev_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) memcpy(new_bus->range + i, bus->range + i + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) flex_array_size(new_bus, range, new_bus->dev_count - i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) synchronize_srcu_expedited(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) /* Destroy the old bus _after_ installing the (null) bus. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) if (!new_bus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) pr_err("kvm: failed to shrink bus, removing it completely\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) for (j = 0; j < bus->dev_count; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) if (j == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) kvm_iodevice_destructor(bus->range[j].dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) kfree(bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) return new_bus ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) gpa_t addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) struct kvm_io_bus *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) int dev_idx, srcu_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) struct kvm_io_device *iodev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) srcu_idx = srcu_read_lock(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) if (!bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) if (dev_idx < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) iodev = bus->range[dev_idx].dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) srcu_read_unlock(&kvm->srcu, srcu_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) return iodev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) static int kvm_debugfs_open(struct inode *inode, struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) int (*get)(void *, u64 *), int (*set)(void *, u64),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) const char *fmt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) inode->i_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) /* The debugfs files are a reference to the kvm struct which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) * is still valid when kvm_destroy_vm is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) * To avoid the race between open and the removal of the debugfs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) * directory we test against the users count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) if (simple_attr_open(inode, file, get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) ? set : NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) fmt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) kvm_put_kvm(stat_data->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) static int kvm_debugfs_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) inode->i_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) simple_attr_release(inode, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) kvm_put_kvm(stat_data->kvm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) *val = *(ulong *)((void *)kvm + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) *(ulong *)((void *)kvm + offset) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) *val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) kvm_for_each_vcpu(i, vcpu, kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) *val += *(u64 *)((void *)vcpu + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) kvm_for_each_vcpu(i, vcpu, kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) *(u64 *)((void *)vcpu + offset) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) static int kvm_stat_data_get(void *data, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) int r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) switch (stat_data->dbgfs_item->kind) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) case KVM_STAT_VM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) r = kvm_get_stat_per_vm(stat_data->kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) stat_data->dbgfs_item->offset, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) case KVM_STAT_VCPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) r = kvm_get_stat_per_vcpu(stat_data->kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) stat_data->dbgfs_item->offset, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) static int kvm_stat_data_clear(void *data, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) int r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) switch (stat_data->dbgfs_item->kind) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) case KVM_STAT_VM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) r = kvm_clear_stat_per_vm(stat_data->kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) stat_data->dbgfs_item->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) case KVM_STAT_VCPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) r = kvm_clear_stat_per_vcpu(stat_data->kvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) stat_data->dbgfs_item->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) static int kvm_stat_data_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) __simple_attr_check_format("%llu\n", 0ull);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) return kvm_debugfs_open(inode, file, kvm_stat_data_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) kvm_stat_data_clear, "%llu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) static const struct file_operations stat_fops_per_vm = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) .open = kvm_stat_data_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) .release = kvm_debugfs_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) .read = simple_attr_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) .write = simple_attr_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) .llseek = no_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) static int vm_stat_get(void *_offset, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) unsigned offset = (long)_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) u64 tmp_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) *val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) list_for_each_entry(kvm, &vm_list, vm_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) kvm_get_stat_per_vm(kvm, offset, &tmp_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) *val += tmp_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) static int vm_stat_clear(void *_offset, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) unsigned offset = (long)_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) list_for_each_entry(kvm, &vm_list, vm_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) kvm_clear_stat_per_vm(kvm, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) static int vcpu_stat_get(void *_offset, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) unsigned offset = (long)_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) u64 tmp_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) *val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) list_for_each_entry(kvm, &vm_list, vm_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) *val += tmp_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) static int vcpu_stat_clear(void *_offset, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) unsigned offset = (long)_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) list_for_each_entry(kvm, &vm_list, vm_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) kvm_clear_stat_per_vcpu(kvm, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) "%llu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) static const struct file_operations *stat_fops[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) [KVM_STAT_VCPU] = &vcpu_stat_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) [KVM_STAT_VM] = &vm_stat_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) struct kobj_uevent_env *env;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) unsigned long long created, active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) if (!kvm_dev.this_device || !kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) mutex_lock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) if (type == KVM_EVENT_CREATE_VM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) kvm_createvm_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) kvm_active_vms++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) } else if (type == KVM_EVENT_DESTROY_VM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) kvm_active_vms--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) created = kvm_createvm_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) active = kvm_active_vms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) mutex_unlock(&kvm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) env = kzalloc(sizeof(*env), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) if (!env)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) add_uevent_var(env, "CREATED=%llu", created);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) add_uevent_var(env, "COUNT=%llu", active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) if (type == KVM_EVENT_CREATE_VM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) add_uevent_var(env, "EVENT=create");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) kvm->userspace_pid = task_pid_nr(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) } else if (type == KVM_EVENT_DESTROY_VM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) add_uevent_var(env, "EVENT=destroy");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) add_uevent_var(env, "PID=%d", kvm->userspace_pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) if (kvm->debugfs_dentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) if (!IS_ERR(tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) add_uevent_var(env, "STATS_PATH=%s", tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) kfree(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) /* no need for checks, since we are adding at most only 5 keys */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) env->envp[env->envp_idx++] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) kfree(env);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) static void kvm_init_debug(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) struct kvm_stats_debugfs_item *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) kvm_debugfs_num_entries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) kvm_debugfs_dir, (void *)(long)p->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) stat_fops[p->kind]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) static int kvm_suspend(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) if (kvm_usage_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) hardware_disable_nolock(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) static void kvm_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) if (kvm_usage_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) #ifdef CONFIG_LOCKDEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) WARN_ON(lockdep_is_held(&kvm_count_lock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) hardware_enable_nolock(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) static struct syscore_ops kvm_syscore_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) .suspend = kvm_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) .resume = kvm_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) return container_of(pn, struct kvm_vcpu, preempt_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) WRITE_ONCE(vcpu->preempted, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) WRITE_ONCE(vcpu->ready, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) __this_cpu_write(kvm_running_vcpu, vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) kvm_arch_sched_in(vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) kvm_arch_vcpu_load(vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) static void kvm_sched_out(struct preempt_notifier *pn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) if (current->state == TASK_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) WRITE_ONCE(vcpu->preempted, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) WRITE_ONCE(vcpu->ready, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) kvm_arch_vcpu_put(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) __this_cpu_write(kvm_running_vcpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) * kvm_get_running_vcpu - get the vcpu running on the current CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) * We can disable preemption locally around accessing the per-CPU variable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) * and use the resolved vcpu pointer after enabling preemption again,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) * because even if the current thread is migrated to another CPU, reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) * the per-CPU value later will give us the same value as we update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824) * per-CPU variable in the preempt notifier handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) struct kvm_vcpu *kvm_get_running_vcpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) struct kvm_vcpu *vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) vcpu = __this_cpu_read(kvm_running_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) return vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) EXPORT_SYMBOL_GPL(kvm_get_running_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841) struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) return &kvm_running_vcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) struct kvm_cpu_compat_check {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) void *opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) int *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) static void check_processor_compat(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) struct kvm_cpu_compat_check *c = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) *c->ret = kvm_arch_check_processor_compat(c->opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) struct module *module)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) struct kvm_cpu_compat_check c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) r = kvm_arch_init(opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) goto out_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) * kvm_arch_init makes sure there's at most one caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) * for architectures that support multiple implementations,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) * like intel and amd on x86.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) * conflicts in case kvm is already setup for another implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) r = kvm_irqfd_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) goto out_irqfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) goto out_free_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) r = kvm_arch_hardware_setup(opaque);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) goto out_free_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) c.ret = &r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) c.opaque = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892) smp_call_function_single(cpu, check_processor_compat, &c, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) goto out_free_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) kvm_starting_cpu, kvm_dying_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) goto out_free_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) register_reboot_notifier(&kvm_reboot_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) /* A kmem cache lets us meet the alignment requirements of fx_save. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) if (!vcpu_align)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) vcpu_align = __alignof__(struct kvm_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) kvm_vcpu_cache =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) SLAB_ACCOUNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) offsetof(struct kvm_vcpu, arch),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) sizeof_field(struct kvm_vcpu, arch),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) if (!kvm_vcpu_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) r = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) goto out_free_3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) r = kvm_async_pf_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) if (r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) kvm_chardev_ops.owner = module;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) kvm_vm_fops.owner = module;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) kvm_vcpu_fops.owner = module;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) r = misc_register(&kvm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) if (r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) pr_err("kvm: misc device register failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) goto out_unreg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931) register_syscore_ops(&kvm_syscore_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) kvm_preempt_ops.sched_in = kvm_sched_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) kvm_preempt_ops.sched_out = kvm_sched_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) kvm_init_debug();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) r = kvm_vfio_ops_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) WARN_ON(r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) out_unreg:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) kvm_async_pf_deinit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) kmem_cache_destroy(kvm_vcpu_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) out_free_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) unregister_reboot_notifier(&kvm_reboot_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) out_free_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) kvm_arch_hardware_unsetup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) out_free_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) free_cpumask_var(cpus_hardware_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) out_free_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) kvm_irqfd_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) out_irqfd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957) kvm_arch_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) out_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) EXPORT_SYMBOL_GPL(kvm_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) void kvm_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) debugfs_remove_recursive(kvm_debugfs_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) misc_deregister(&kvm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) kmem_cache_destroy(kvm_vcpu_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) kvm_async_pf_deinit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) unregister_syscore_ops(&kvm_syscore_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) unregister_reboot_notifier(&kvm_reboot_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) on_each_cpu(hardware_disable_nolock, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) kvm_arch_hardware_unsetup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) kvm_arch_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) kvm_irqfd_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) free_cpumask_var(cpus_hardware_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) kvm_vfio_ops_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) EXPORT_SYMBOL_GPL(kvm_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) struct kvm_vm_worker_thread_context {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) struct kvm *kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) struct task_struct *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) struct completion init_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) kvm_vm_thread_fn_t thread_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) uintptr_t data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) static int kvm_vm_worker_thread(void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) * The init_context is allocated on the stack of the parent thread, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) * we have to locally copy anything that is needed beyond initialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) struct kvm_vm_worker_thread_context *init_context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) struct kvm *kvm = init_context->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) kvm_vm_thread_fn_t thread_fn = init_context->thread_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) uintptr_t data = init_context->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) err = kthread_park(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) /* kthread_park(current) is never supposed to return an error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) WARN_ON(err != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) goto init_complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) err = cgroup_attach_task_all(init_context->parent, current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) kvm_err("%s: cgroup_attach_task_all failed with err %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) __func__, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) goto init_complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) set_user_nice(current, task_nice(init_context->parent));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) init_complete:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) init_context->err = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) complete(&init_context->init_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) init_context = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) /* Wait to be woken up by the spawner before proceeding. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) kthread_parkme();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) if (!kthread_should_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) err = thread_fn(kvm, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) int kvm_vm_create_worker_thread(struct kvm *kvm, kvm_vm_thread_fn_t thread_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) uintptr_t data, const char *name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) struct task_struct **thread_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) struct kvm_vm_worker_thread_context init_context = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) struct task_struct *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) *thread_ptr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042) init_context.kvm = kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) init_context.parent = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) init_context.thread_fn = thread_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) init_context.data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) init_completion(&init_context.init_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) thread = kthread_run(kvm_vm_worker_thread, &init_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) "%s-%d", name, task_pid_nr(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) if (IS_ERR(thread))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) return PTR_ERR(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) /* kthread_run is never supposed to return NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) WARN_ON(thread == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) wait_for_completion(&init_context.init_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) if (!init_context.err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) *thread_ptr = thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) return init_context.err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags