^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Support cstate residency counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2015, Intel Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author: Kan Liang (kan.liang@intel.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This library is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * modify it under the terms of the GNU Library General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * License as published by the Free Software Foundation; either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * version 2 of the License, or (at your option) any later version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * This library is distributed in the hope that it will be useful,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * but WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Library General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * This file export cstate related free running (read-only) counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * for perf. These counters may be use simultaneously by other tools,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * such as turbostat. However, it still make sense to implement them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * in perf. Because we can conveniently collect them together with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * other events, and allow to use them from tools without special MSR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * access code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * The events only support system-wide mode counting. There is no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * sampling support because it is not supported by the hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * According to counters' scope and category, two PMUs are registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * with the perf_event core subsystem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * - 'cstate_core': The counter is available for each physical core.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * The counters include CORE_C*_RESIDENCY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * - 'cstate_pkg': The counter is available for each physical package.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * The counters include PKG_C*_RESIDENCY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * All of these counters are specified in the Intel® 64 and IA-32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Architectures Software Developer.s Manual Vol3b.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * Model specific counters:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * MSR_CORE_C1_RES: CORE C1 Residency Counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * perf code: 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * Available model: SLM,AMT,GLM,CNL,TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * Scope: Core (each processor core has a MSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * perf code: 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * CNL,KBL,CML,TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * Scope: Core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * perf code: 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * SKL,KNL,GLM,CNL,KBL,CML,ICL,TGL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * Scope: Core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * perf code: 0x03
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * ICL,TGL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Scope: Core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * perf code: 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * KBL,CML,ICL,TGL,TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * perf code: 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * GLM,CNL,KBL,CML,ICL,TGL,TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * perf code: 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * SKL,KNL,GLM,CNL,KBL,CML,ICL,TGL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * perf code: 0x03
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * KBL,CML,ICL,TGL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * perf code: 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * perf code: 0x05
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * perf code: 0x06
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * TNT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Scope: Package (physical package)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #include <linux/nospec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #include <asm/intel-family.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #include "../perf_event.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #include "../probe.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static ssize_t __cstate_##_var##_show(struct device *dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct device_attribute *attr, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) char *page) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return sprintf(page, _format "\n"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static struct device_attribute format_attr_##_var = \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static ssize_t cstate_get_attr_cpumask(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) char *buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Model -> events mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct cstate_model {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long core_events;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) unsigned long pkg_events;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned long quirks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /* Quirk flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define KNL_CORE_C6_MSR (1UL << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct perf_cstate_msr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) u64 msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct perf_pmu_events_attr *attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /* cstate_core PMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static struct pmu cstate_core_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static bool has_cstate_core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) enum perf_cstate_core_events {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) PERF_CSTATE_CORE_C1_RES = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) PERF_CSTATE_CORE_C3_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) PERF_CSTATE_CORE_C6_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) PERF_CSTATE_CORE_C7_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) PERF_CSTATE_CORE_EVENT_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static unsigned long core_msr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) PMU_EVENT_GROUP(events, cstate_core_c1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) PMU_EVENT_GROUP(events, cstate_core_c3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) PMU_EVENT_GROUP(events, cstate_core_c6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) PMU_EVENT_GROUP(events, cstate_core_c7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static bool test_msr(int idx, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return test_bit(idx, (unsigned long *) data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static struct perf_msr core_msr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &group_cstate_core_c3, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &group_cstate_core_c6, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &group_cstate_core_c7, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) static struct attribute *attrs_empty[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * There are no default events, but we need to create
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * "events" group (with empty attrs) before updating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * it with detected events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static struct attribute_group core_events_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) .name = "events",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .attrs = attrs_empty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static struct attribute *core_format_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) &format_attr_core_event.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) static struct attribute_group core_format_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) .name = "format",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) .attrs = core_format_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static cpumask_t cstate_core_cpu_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static struct attribute *cstate_cpumask_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) &dev_attr_cpumask.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static struct attribute_group cpumask_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) .attrs = cstate_cpumask_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) static const struct attribute_group *core_attr_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) &core_events_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) &core_format_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) &cpumask_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /* cstate_pkg PMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static struct pmu cstate_pkg_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) static bool has_cstate_pkg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) enum perf_cstate_pkg_events {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) PERF_CSTATE_PKG_C2_RES = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) PERF_CSTATE_PKG_C3_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) PERF_CSTATE_PKG_C6_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) PERF_CSTATE_PKG_C7_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) PERF_CSTATE_PKG_C8_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) PERF_CSTATE_PKG_C9_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) PERF_CSTATE_PKG_C10_RES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) PERF_CSTATE_PKG_EVENT_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) static unsigned long pkg_msr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) PMU_EVENT_GROUP(events, cstate_pkg_c2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) PMU_EVENT_GROUP(events, cstate_pkg_c3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) PMU_EVENT_GROUP(events, cstate_pkg_c6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) PMU_EVENT_GROUP(events, cstate_pkg_c7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) PMU_EVENT_GROUP(events, cstate_pkg_c8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) PMU_EVENT_GROUP(events, cstate_pkg_c9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) PMU_EVENT_GROUP(events, cstate_pkg_c10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static struct perf_msr pkg_msr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &group_cstate_pkg_c2, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &group_cstate_pkg_c3, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &group_cstate_pkg_c6, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &group_cstate_pkg_c7, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &group_cstate_pkg_c8, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &group_cstate_pkg_c9, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &group_cstate_pkg_c10, test_msr },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static struct attribute_group pkg_events_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) .name = "events",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) .attrs = attrs_empty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static struct attribute *pkg_format_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) &format_attr_pkg_event.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static struct attribute_group pkg_format_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) .name = "format",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) .attrs = pkg_format_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static cpumask_t cstate_pkg_cpu_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static const struct attribute_group *pkg_attr_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) &pkg_events_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) &pkg_format_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) &cpumask_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static ssize_t cstate_get_attr_cpumask(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct pmu *pmu = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (pmu == &cstate_core_pmu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) else if (pmu == &cstate_pkg_pmu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static int cstate_pmu_event_init(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) u64 cfg = event->attr.config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (event->attr.type != event->pmu->type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* unsupported modes and filters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (event->attr.sample_period) /* no sampling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (event->cpu < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (event->pmu == &cstate_core_pmu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (!(core_msr_mask & (1 << cfg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) event->hw.event_base = core_msr[cfg].msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) cpu = cpumask_any_and(&cstate_core_cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) topology_sibling_cpumask(event->cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) } else if (event->pmu == &cstate_pkg_pmu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!(pkg_msr_mask & (1 << cfg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) event->hw.event_base = pkg_msr[cfg].msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) topology_die_cpumask(event->cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (cpu >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) event->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) event->hw.config = cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) event->hw.idx = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static inline u64 cstate_pmu_read_counter(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) rdmsrl(event->hw.event_base, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static void cstate_pmu_event_update(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) u64 prev_raw_count, new_raw_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) prev_raw_count = local64_read(&hwc->prev_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) new_raw_count = cstate_pmu_read_counter(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) new_raw_count) != prev_raw_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) local64_add(new_raw_count - prev_raw_count, &event->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) static void cstate_pmu_event_start(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static void cstate_pmu_event_stop(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) cstate_pmu_event_update(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) static void cstate_pmu_event_del(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) cstate_pmu_event_stop(event, PERF_EF_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static int cstate_pmu_event_add(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (mode & PERF_EF_START)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) cstate_pmu_event_start(event, mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * Check if exiting cpu is the designated reader. If so migrate the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * events when there is a valid target available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static int cstate_cpu_exit(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) unsigned int target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (has_cstate_core &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /* Migrate events if there is a valid target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (target < nr_cpu_ids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) cpumask_set_cpu(target, &cstate_core_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) perf_pmu_migrate_context(&cstate_core_pmu, cpu, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (has_cstate_pkg &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) /* Migrate events if there is a valid target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (target < nr_cpu_ids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) cpumask_set_cpu(target, &cstate_pkg_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) static int cstate_cpu_init(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) unsigned int target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * If this is the first online thread of that core, set it in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * the core cpu mask as the designated reader.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) target = cpumask_any_and(&cstate_core_cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) topology_sibling_cpumask(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (has_cstate_core && target >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) cpumask_set_cpu(cpu, &cstate_core_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * If this is the first online thread of that package, set it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * in the package cpu mask as the designated reader.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) target = cpumask_any_and(&cstate_pkg_cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) topology_die_cpumask(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (has_cstate_pkg && target >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static const struct attribute_group *core_attr_update[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) &group_cstate_core_c1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) &group_cstate_core_c3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) &group_cstate_core_c6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) &group_cstate_core_c7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static const struct attribute_group *pkg_attr_update[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) &group_cstate_pkg_c2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) &group_cstate_pkg_c3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) &group_cstate_pkg_c6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) &group_cstate_pkg_c7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) &group_cstate_pkg_c8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) &group_cstate_pkg_c9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) &group_cstate_pkg_c10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) static struct pmu cstate_core_pmu = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) .attr_groups = core_attr_groups,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) .attr_update = core_attr_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) .name = "cstate_core",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) .task_ctx_nr = perf_invalid_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) .event_init = cstate_pmu_event_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) .add = cstate_pmu_event_add,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) .del = cstate_pmu_event_del,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) .start = cstate_pmu_event_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) .stop = cstate_pmu_event_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) .read = cstate_pmu_event_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static struct pmu cstate_pkg_pmu = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) .attr_groups = pkg_attr_groups,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) .attr_update = pkg_attr_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) .name = "cstate_pkg",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) .task_ctx_nr = perf_invalid_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) .event_init = cstate_pmu_event_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) .add = cstate_pmu_event_add,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) .del = cstate_pmu_event_del,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) .start = cstate_pmu_event_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) .stop = cstate_pmu_event_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) .read = cstate_pmu_event_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) .module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static const struct cstate_model nhm_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) BIT(PERF_CSTATE_CORE_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) BIT(PERF_CSTATE_PKG_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static const struct cstate_model snb_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) BIT(PERF_CSTATE_CORE_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) BIT(PERF_CSTATE_CORE_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) BIT(PERF_CSTATE_PKG_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) static const struct cstate_model hswult_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) BIT(PERF_CSTATE_CORE_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) BIT(PERF_CSTATE_CORE_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) BIT(PERF_CSTATE_PKG_C7_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) BIT(PERF_CSTATE_PKG_C8_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) BIT(PERF_CSTATE_PKG_C9_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) BIT(PERF_CSTATE_PKG_C10_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) static const struct cstate_model cnl_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) BIT(PERF_CSTATE_CORE_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) BIT(PERF_CSTATE_CORE_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) BIT(PERF_CSTATE_CORE_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) BIT(PERF_CSTATE_PKG_C7_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) BIT(PERF_CSTATE_PKG_C8_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) BIT(PERF_CSTATE_PKG_C9_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) BIT(PERF_CSTATE_PKG_C10_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static const struct cstate_model icl_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) .core_events = BIT(PERF_CSTATE_CORE_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) BIT(PERF_CSTATE_CORE_C7_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) BIT(PERF_CSTATE_PKG_C7_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) BIT(PERF_CSTATE_PKG_C8_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) BIT(PERF_CSTATE_PKG_C9_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) BIT(PERF_CSTATE_PKG_C10_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) static const struct cstate_model slm_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) BIT(PERF_CSTATE_CORE_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) .quirks = SLM_PKG_C6_USE_C7_MSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static const struct cstate_model knl_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) .core_events = BIT(PERF_CSTATE_CORE_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) BIT(PERF_CSTATE_PKG_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .quirks = KNL_CORE_C6_MSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static const struct cstate_model glm_cstates __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) BIT(PERF_CSTATE_CORE_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) BIT(PERF_CSTATE_CORE_C6_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) BIT(PERF_CSTATE_PKG_C3_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) BIT(PERF_CSTATE_PKG_C6_RES) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) BIT(PERF_CSTATE_PKG_C10_RES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static const struct x86_cpu_id intel_cstates_match[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hswult_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &slm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &slm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &slm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &snb_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &hswult_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &hswult_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &hswult_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &hswult_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &cnl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &glm_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &icl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &icl_cstates),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) { },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) static int __init cstate_probe(const struct cstate_model *cm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* SLM has different MSR for PKG C6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /* KNL has different MSR for CORE C6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (cm->quirks & KNL_CORE_C6_MSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) core_msr_mask = perf_msr_probe(core_msr, PERF_CSTATE_CORE_EVENT_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) true, (void *) &cm->core_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) true, (void *) &cm->pkg_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) has_cstate_core = !!core_msr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) has_cstate_pkg = !!pkg_msr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) static inline void cstate_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (has_cstate_core)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) perf_pmu_unregister(&cstate_core_pmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (has_cstate_pkg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) perf_pmu_unregister(&cstate_pkg_pmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static int __init cstate_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_STARTING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) "perf/x86/cstate:starting", cstate_cpu_init, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_ONLINE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) "perf/x86/cstate:online", NULL, cstate_cpu_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (has_cstate_core) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) has_cstate_core = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) pr_info("Failed to register cstate core pmu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) cstate_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (has_cstate_pkg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) if (topology_max_die_per_package() > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) err = perf_pmu_register(&cstate_pkg_pmu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) "cstate_die", -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) err = perf_pmu_register(&cstate_pkg_pmu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) cstate_pkg_pmu.name, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) has_cstate_pkg = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) pr_info("Failed to register cstate pkg pmu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) cstate_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) static int __init cstate_pmu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) const struct x86_cpu_id *id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) id = x86_match_cpu(intel_cstates_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (!id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) err = cstate_probe((const struct cstate_model *) id->driver_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return cstate_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) module_init(cstate_pmu_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) static void __exit cstate_pmu_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) cstate_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) module_exit(cstate_pmu_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags