^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel-based Virtual Machine -- Performance Monitoring Unit support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright 2015 Red Hat, Inc. and/or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Avi Kivity <avi@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Gleb Natapov <gleb@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Wei Huang <wei@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "cpuid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "lapic.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "pmu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) /* This is enough to filter the vast majority of currently defined events. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* NOTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - Each perf counter is defined as "struct kvm_pmc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * - There are two types of perf counters: general purpose (gp) and fixed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * gp counters are stored in gp_counters[] and fixed counters are stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * in fixed_counters[] respectively. Both of them are part of "struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * kvm_pmu";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * - pmu.c understands the difference between gp counters and fixed counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * However AMD doesn't support fixed-counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * - There are three types of index to access perf counters (PMC):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * has MSR_K7_PERFCTRn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * 2. MSR Index (named idx): This normally is used by RDPMC instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * that it also supports fixed counters. idx can be used to as index to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * gp and fixed counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * 3. Global PMC Index (named pmc): pmc is an index specific to PMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * code. Each pmc, stored in kvm_pmc.idx field, is unique across
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * all perf counters (both gp and fixed). The mapping relationship
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * between pmc and perf counters is as the following:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * * Intel: [0 .. INTEL_PMC_MAX_GENERIC-1] <=> gp counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static void kvm_pmi_trigger_fn(struct irq_work *irq_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) kvm_pmu_deliver_pmi(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static void kvm_perf_overflow(struct perf_event *perf_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct kvm_pmc *pmc = perf_event->overflow_handler_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct kvm_pmu *pmu = pmc_to_pmu(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (!test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static void kvm_perf_overflow_intr(struct perf_event *perf_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct kvm_pmc *pmc = perf_event->overflow_handler_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct kvm_pmu *pmu = pmc_to_pmu(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!test_and_set_bit(pmc->idx, pmu->reprogram_pmi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Inject PMI. If vcpu was in a guest mode during NMI PMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * can be ejected on a guest mode re-entry. Otherwise we can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * be sure that vcpu wasn't executing hlt instruction at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * time of vmexit and is not going to re-enter guest mode until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * woken up. So we should wake it, but this is impossible from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * NMI context. Do it from irq work instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!kvm_is_in_guest())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) irq_work_queue(&pmc_to_pmu(pmc)->irq_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static void pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) u64 config, bool exclude_user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) bool exclude_kernel, bool intr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) bool in_tx, bool in_tx_cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct perf_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct perf_event_attr attr = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .type = type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) .size = sizeof(attr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) .pinned = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) .exclude_idle = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) .exclude_host = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) .exclude_user = exclude_user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) .exclude_kernel = exclude_kernel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .config = config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) attr.sample_period = get_sample_period(pmc, pmc->counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (in_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) attr.config |= HSW_IN_TX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (in_tx_cp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * period. Just clear the sample period so at least
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * allocating the counter doesn't fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) attr.sample_period = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) attr.config |= HSW_IN_TX_CHECKPOINTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) event = perf_event_create_kernel_counter(&attr, -1, current,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) intr ? kvm_perf_overflow_intr :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) kvm_perf_overflow, pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (IS_ERR(event)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) PTR_ERR(event), pmc->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) pmc->perf_event = event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) pmc_to_pmu(pmc)->event_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) clear_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static void pmc_pause_counter(struct kvm_pmc *pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u64 counter = pmc->counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (!pmc->perf_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* update counter, reset event value to avoid redundant accumulation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) counter += perf_event_pause(pmc->perf_event, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) pmc->counter = counter & pmc_bitmask(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static bool pmc_resume_counter(struct kvm_pmc *pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (!pmc->perf_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* recalibrate sample period and check if it's accepted by perf core */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (perf_event_period(pmc->perf_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) get_sample_period(pmc, pmc->counter)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /* reuse perf_event to serve as pmc_reprogram_counter() does*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) perf_event_enable(pmc->perf_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) u64 config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) u32 type = PERF_TYPE_RAW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct kvm *kvm = pmc->vcpu->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct kvm_pmu_event_filter *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) bool allow_event = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) printk_once("kvm pmu: pin control bit is ignored\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) pmc->eventsel = eventsel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pmc_pause_counter(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (!(eventsel & ARCH_PERFMON_EVENTSEL_ENABLE) || !pmc_is_enabled(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (filter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) for (i = 0; i < filter->nevents; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (filter->events[i] ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) (eventsel & AMD64_RAW_EVENT_MASK_NB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (filter->action == KVM_PMU_EVENT_ALLOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) i == filter->nevents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) allow_event = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (filter->action == KVM_PMU_EVENT_DENY &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) i < filter->nevents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) allow_event = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (!allow_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (!(eventsel & (ARCH_PERFMON_EVENTSEL_EDGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ARCH_PERFMON_EVENTSEL_INV |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ARCH_PERFMON_EVENTSEL_CMASK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) HSW_IN_TX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) HSW_IN_TX_CHECKPOINTED))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) config = kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (config != PERF_COUNT_HW_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) type = PERF_TYPE_HARDWARE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (type == PERF_TYPE_RAW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) config = eventsel & AMD64_RAW_EVENT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (pmc->current_config == eventsel && pmc_resume_counter(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) pmc_release_perf_event(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pmc->current_config = eventsel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) pmc_reprogram_counter(pmc, type, config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) eventsel & ARCH_PERFMON_EVENTSEL_INT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) (eventsel & HSW_IN_TX),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) (eventsel & HSW_IN_TX_CHECKPOINTED));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) EXPORT_SYMBOL_GPL(reprogram_gp_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) unsigned en_field = ctrl & 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) bool pmi = ctrl & 0x8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct kvm_pmu_event_filter *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) struct kvm *kvm = pmc->vcpu->kvm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) pmc_pause_counter(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (!en_field || !pmc_is_enabled(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (filter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (filter->action == KVM_PMU_EVENT_DENY &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) test_bit(idx, (ulong *)&filter->fixed_counter_bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (filter->action == KVM_PMU_EVENT_ALLOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) !test_bit(idx, (ulong *)&filter->fixed_counter_bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (pmc->current_config == (u64)ctrl && pmc_resume_counter(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pmc_release_perf_event(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) pmc->current_config = (u64)ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) kvm_x86_ops.pmu_ops->find_fixed_event(idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) !(en_field & 0x2), /* exclude user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) !(en_field & 0x1), /* exclude kernel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) pmi, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) EXPORT_SYMBOL_GPL(reprogram_fixed_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (!pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (pmc_is_gp(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) reprogram_gp_counter(pmc, pmc->eventsel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int idx = pmc_idx - INTEL_PMC_IDX_FIXED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) u8 ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) reprogram_fixed_counter(pmc, ctrl, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) EXPORT_SYMBOL_GPL(reprogram_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) int bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (unlikely(!pmc || !pmc->perf_event)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) clear_bit(bit, pmu->reprogram_pmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) reprogram_counter(pmu, bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * Unused perf_events are only released if the corresponding MSRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * weren't accessed during the last vCPU time slice. kvm_arch_sched_in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * triggers KVM_REQ_PMU if cleanup is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (unlikely(pmu->need_cleanup))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) kvm_pmu_cleanup(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /* check if idx is a valid index to access PMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) bool is_vmware_backdoor_pmc(u32 pmc_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) switch (pmc_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) case VMWARE_BACKDOOR_PMC_HOST_TSC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) case VMWARE_BACKDOOR_PMC_REAL_TIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) u64 ctr_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) switch (idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) case VMWARE_BACKDOOR_PMC_HOST_TSC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) ctr_val = rdtsc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) case VMWARE_BACKDOOR_PMC_REAL_TIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) ctr_val = ktime_get_boottime_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) ctr_val = ktime_get_boottime_ns() +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) vcpu->kvm->arch.kvmclock_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) *data = ctr_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) bool fast_mode = idx & (1u << 31);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct kvm_pmc *pmc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) u64 mask = fast_mode ? ~0u : ~0ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (!pmu->version)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (is_vmware_backdoor_pmc(idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (!pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (!(kvm_read_cr4(vcpu) & X86_CR4_PCE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) (kvm_x86_ops.get_cpl(vcpu) != 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) (kvm_read_cr0(vcpu) & X86_CR0_PE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) *data = pmc_read_counter(pmc) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (lapic_in_kernel(vcpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) __set_bit(pmc->idx, pmu->pmc_in_use);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /* refresh PMU settings. This function generally is called when underlying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * settings are changed (such as changes of PMU CPUID by guest VMs), which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * should rarely happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) kvm_x86_ops.pmu_ops->refresh(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) void kvm_pmu_reset(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) irq_work_sync(&pmu->irq_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) kvm_x86_ops.pmu_ops->reset(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) void kvm_pmu_init(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) memset(pmu, 0, sizeof(*pmu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) kvm_x86_ops.pmu_ops->init(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) pmu->event_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) pmu->need_cleanup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) kvm_pmu_refresh(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static inline bool pmc_speculative_in_use(struct kvm_pmc *pmc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct kvm_pmu *pmu = pmc_to_pmu(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) if (pmc_is_fixed(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return fixed_ctrl_field(pmu->fixed_ctr_ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) pmc->idx - INTEL_PMC_IDX_FIXED) & 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return pmc->eventsel & ARCH_PERFMON_EVENTSEL_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /* Release perf_events for vPMCs that have been unused for a full time slice. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct kvm_pmc *pmc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) pmu->need_cleanup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) pmu->pmc_in_use, X86_PMC_IDX_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) pmc_stop_counter(pmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) kvm_pmu_reset(vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct kvm_pmu_event_filter tmp, *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (copy_from_user(&tmp, argp, sizeof(tmp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (tmp.action != KVM_PMU_EVENT_ALLOW &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) tmp.action != KVM_PMU_EVENT_DENY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (tmp.flags != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) size = struct_size(filter, events, tmp.nevents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) filter = kmalloc(size, GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (!filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) r = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (copy_from_user(filter, argp, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) goto cleanup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* Ensure nevents can't be changed between the user copies. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) *filter = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) mutex_lock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) mutex_is_locked(&kvm->lock));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) mutex_unlock(&kvm->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) synchronize_srcu_expedited(&kvm->srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) r = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) cleanup:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) kfree(filter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags