^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * local apic based NMI watchdog for various CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This file also handles reservation of performance counters for coordination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * with other users (like oprofile).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Note that these events normally don't tick when the CPU idles. This means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * the frequency varies with CPU load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Original code for K7/P6 written by Keith Owens
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/apic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * offset from MSR_P4_BSU_ESCR0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * It will be the max for all platforms (for now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define NMI_MAX_COUNTER_BITS 66
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * perfctr_nmi_owner tracks the ownership of the perfctr registers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * evtsel_nmi_owner tracks the ownership of the event selection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * - different performance counters/ event selection may be reserved for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * different subsystems this reservation system just tries to coordinate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * things a little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /* converts an msr to an appropriate reservation bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* returns the bit offset of the performance counter register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) switch (boot_cpu_data.x86_vendor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) case X86_VENDOR_HYGON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) case X86_VENDOR_AMD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (msr >= MSR_F15H_PERF_CTR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return (msr - MSR_F15H_PERF_CTR) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return msr - MSR_K7_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) case X86_VENDOR_INTEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return msr - MSR_ARCH_PERFMON_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) switch (boot_cpu_data.x86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) return msr - MSR_P6_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) case 11:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) return msr - MSR_KNC_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) case 15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return msr - MSR_P4_BPU_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) case X86_VENDOR_ZHAOXIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) case X86_VENDOR_CENTAUR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return msr - MSR_ARCH_PERFMON_PERFCTR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * converts an msr to an appropriate reservation bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * returns the bit offset of the event selection register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* returns the bit offset of the event selection register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) switch (boot_cpu_data.x86_vendor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) case X86_VENDOR_HYGON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) case X86_VENDOR_AMD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (msr >= MSR_F15H_PERF_CTL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return (msr - MSR_F15H_PERF_CTL) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return msr - MSR_K7_EVNTSEL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) case X86_VENDOR_INTEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return msr - MSR_ARCH_PERFMON_EVENTSEL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) switch (boot_cpu_data.x86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return msr - MSR_P6_EVNTSEL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) case 11:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return msr - MSR_KNC_EVNTSEL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) case 15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return msr - MSR_P4_BSU_ESCR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) case X86_VENDOR_ZHAOXIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) case X86_VENDOR_CENTAUR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return msr - MSR_ARCH_PERFMON_EVENTSEL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* checks for a bit availability (hack for oprofile) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) BUG_ON(counter > NMI_MAX_COUNTER_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return !test_bit(counter, perfctr_nmi_owner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) int reserve_perfctr_nmi(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned int counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) counter = nmi_perfctr_msr_to_bit(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /* register not managed by the allocator? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (counter > NMI_MAX_COUNTER_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (!test_and_set_bit(counter, perfctr_nmi_owner))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) EXPORT_SYMBOL(reserve_perfctr_nmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) void release_perfctr_nmi(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) unsigned int counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) counter = nmi_perfctr_msr_to_bit(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* register not managed by the allocator? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (counter > NMI_MAX_COUNTER_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) clear_bit(counter, perfctr_nmi_owner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) EXPORT_SYMBOL(release_perfctr_nmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) int reserve_evntsel_nmi(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) unsigned int counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) counter = nmi_evntsel_msr_to_bit(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* register not managed by the allocator? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (counter > NMI_MAX_COUNTER_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!test_and_set_bit(counter, evntsel_nmi_owner))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) EXPORT_SYMBOL(reserve_evntsel_nmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) void release_evntsel_nmi(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned int counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) counter = nmi_evntsel_msr_to_bit(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* register not managed by the allocator? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (counter > NMI_MAX_COUNTER_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) clear_bit(counter, evntsel_nmi_owner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) EXPORT_SYMBOL(release_evntsel_nmi);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate