Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /* Driver for Intel Xeon Phi "Knights Corner" PMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <asm/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include "../perf_event.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) static const u64 knc_perfmon_event_map[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)   [PERF_COUNT_HW_CPU_CYCLES]		= 0x002a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)   [PERF_COUNT_HW_INSTRUCTIONS]		= 0x0016,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)   [PERF_COUNT_HW_CACHE_REFERENCES]	= 0x0028,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)   [PERF_COUNT_HW_CACHE_MISSES]		= 0x0029,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= 0x0012,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)   [PERF_COUNT_HW_BRANCH_MISSES]		= 0x002b,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) static const u64 __initconst knc_hw_cache_event_ids
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 				[PERF_COUNT_HW_CACHE_MAX]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 				[PERF_COUNT_HW_CACHE_OP_MAX]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  [ C(L1D) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		/* On Xeon Phi event "0" is a valid DATA_READ          */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 		/*   (L1 Data Cache Reads) Instruction.                */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 		/* We code this as ARCH_PERFMON_EVENTSEL_INT as this   */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 		/* bit will always be set in x86_pmu_hw_config().      */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 		[ C(RESULT_ACCESS) ] = ARCH_PERFMON_EVENTSEL_INT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 						/* DATA_READ           */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 		[ C(RESULT_MISS)   ] = 0x0003,	/* DATA_READ_MISS      */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		[ C(RESULT_ACCESS) ] = 0x0001,	/* DATA_WRITE          */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 		[ C(RESULT_MISS)   ] = 0x0004,	/* DATA_WRITE_MISS     */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 		[ C(RESULT_ACCESS) ] = 0x0011,	/* L1_DATA_PF1         */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 		[ C(RESULT_MISS)   ] = 0x001c,	/* L1_DATA_PF1_MISS    */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  [ C(L1I ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 		[ C(RESULT_ACCESS) ] = 0x000c,	/* CODE_READ          */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 		[ C(RESULT_MISS)   ] = 0x000e,	/* CODE_CACHE_MISS    */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 		[ C(RESULT_ACCESS) ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		[ C(RESULT_MISS)   ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		[ C(RESULT_ACCESS) ] = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 		[ C(RESULT_MISS)   ] = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  [ C(LL  ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		[ C(RESULT_ACCESS) ] = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 		[ C(RESULT_MISS)   ] = 0x10cb,	/* L2_READ_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 		[ C(RESULT_ACCESS) ] = 0x10cc,	/* L2_WRITE_HIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		[ C(RESULT_MISS)   ] = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		[ C(RESULT_ACCESS) ] = 0x10fc,	/* L2_DATA_PF2      */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		[ C(RESULT_MISS)   ] = 0x10fe,	/* L2_DATA_PF2_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  [ C(DTLB) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		[ C(RESULT_ACCESS) ] = ARCH_PERFMON_EVENTSEL_INT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 						/* DATA_READ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 						/* see note on L1 OP_READ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 		[ C(RESULT_MISS)   ] = 0x0002,	/* DATA_PAGE_WALK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		[ C(RESULT_ACCESS) ] = 0x0001,	/* DATA_WRITE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		[ C(RESULT_MISS)   ] = 0x0002,	/* DATA_PAGE_WALK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		[ C(RESULT_ACCESS) ] = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		[ C(RESULT_MISS)   ] = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89)  [ C(ITLB) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		[ C(RESULT_ACCESS) ] = 0x000c,	/* CODE_READ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		[ C(RESULT_MISS)   ] = 0x000d,	/* CODE_PAGE_WALK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		[ C(RESULT_ACCESS) ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		[ C(RESULT_MISS)   ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		[ C(RESULT_ACCESS) ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		[ C(RESULT_MISS)   ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)  [ C(BPU ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	[ C(OP_READ) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		[ C(RESULT_ACCESS) ] = 0x0012,	/* BRANCHES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		[ C(RESULT_MISS)   ] = 0x002b,	/* BRANCHES_MISPREDICTED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	[ C(OP_WRITE) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		[ C(RESULT_ACCESS) ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 		[ C(RESULT_MISS)   ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	[ C(OP_PREFETCH) ] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		[ C(RESULT_ACCESS) ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		[ C(RESULT_MISS)   ] = -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static u64 knc_pmu_event_map(int hw_event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	return knc_perfmon_event_map[hw_event];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) static struct event_constraint knc_event_constraints[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	INTEL_EVENT_CONSTRAINT(0xc3, 0x1),	/* HWP_L2HIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	INTEL_EVENT_CONSTRAINT(0xc4, 0x1),	/* HWP_L2MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	INTEL_EVENT_CONSTRAINT(0xc8, 0x1),	/* L2_READ_HIT_E */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	INTEL_EVENT_CONSTRAINT(0xc9, 0x1),	/* L2_READ_HIT_M */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	INTEL_EVENT_CONSTRAINT(0xca, 0x1),	/* L2_READ_HIT_S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	INTEL_EVENT_CONSTRAINT(0xcb, 0x1),	/* L2_READ_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	INTEL_EVENT_CONSTRAINT(0xcc, 0x1),	/* L2_WRITE_HIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	INTEL_EVENT_CONSTRAINT(0xce, 0x1),	/* L2_STRONGLY_ORDERED_STREAMING_VSTORES_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	INTEL_EVENT_CONSTRAINT(0xcf, 0x1),	/* L2_WEAKLY_ORDERED_STREAMING_VSTORE_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	INTEL_EVENT_CONSTRAINT(0xd7, 0x1),	/* L2_VICTIM_REQ_WITH_DATA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	INTEL_EVENT_CONSTRAINT(0xe3, 0x1),	/* SNP_HITM_BUNIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	INTEL_EVENT_CONSTRAINT(0xe6, 0x1),	/* SNP_HIT_L2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	INTEL_EVENT_CONSTRAINT(0xe7, 0x1),	/* SNP_HITM_L2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	INTEL_EVENT_CONSTRAINT(0xf1, 0x1),	/* L2_DATA_READ_MISS_CACHE_FILL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	INTEL_EVENT_CONSTRAINT(0xf2, 0x1),	/* L2_DATA_WRITE_MISS_CACHE_FILL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	INTEL_EVENT_CONSTRAINT(0xf6, 0x1),	/* L2_DATA_READ_MISS_MEM_FILL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	INTEL_EVENT_CONSTRAINT(0xf7, 0x1),	/* L2_DATA_WRITE_MISS_MEM_FILL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	INTEL_EVENT_CONSTRAINT(0xfc, 0x1),	/* L2_DATA_PF2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	INTEL_EVENT_CONSTRAINT(0xfd, 0x1),	/* L2_DATA_PF2_DROP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	INTEL_EVENT_CONSTRAINT(0xfe, 0x1),	/* L2_DATA_PF2_MISS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	INTEL_EVENT_CONSTRAINT(0xff, 0x1),	/* L2_DATA_HIT_INFLIGHT_PF2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	EVENT_CONSTRAINT_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define MSR_KNC_IA32_PERF_GLOBAL_STATUS		0x0000002d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL	0x0000002e
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define MSR_KNC_IA32_PERF_GLOBAL_CTRL		0x0000002f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define KNC_ENABLE_COUNTER0			0x00000001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define KNC_ENABLE_COUNTER1			0x00000002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static void knc_pmu_disable_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	val &= ~(KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static void knc_pmu_enable_all(int added)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	val |= (KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) knc_pmu_disable_event(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	val = hwc->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	(void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static void knc_pmu_enable_event(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	val = hwc->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	val |= ARCH_PERFMON_EVENTSEL_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	(void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static inline u64 knc_pmu_get_status(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	u64 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static inline void knc_pmu_ack_status(u64 ack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL, ack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static int knc_pmu_handle_irq(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	struct perf_sample_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	struct cpu_hw_events *cpuc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	int handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	int bit, loops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	u64 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	cpuc = this_cpu_ptr(&cpu_hw_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	knc_pmu_disable_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	status = knc_pmu_get_status();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	if (!status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		knc_pmu_enable_all(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	loops = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	knc_pmu_ack_status(status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	if (++loops > 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 		WARN_ONCE(1, "perf: irq loop stuck!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		perf_event_print_debug();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	inc_irq_stat(apic_perf_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		struct perf_event *event = cpuc->events[bit];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		handled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		if (!test_bit(bit, cpuc->active_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 		if (!intel_pmu_save_and_restart(event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 		perf_sample_data_init(&data, 0, event->hw.last_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		if (perf_event_overflow(event, &data, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 			x86_pmu_stop(event, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	 * Repeat if there is more work to be done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	status = knc_pmu_get_status();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	if (status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	/* Only restore PMU state when it's active. See x86_pmu_disable(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	if (cpuc->enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		knc_pmu_enable_all(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) PMU_FORMAT_ATTR(event,	"config:0-7"	);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) PMU_FORMAT_ATTR(umask,	"config:8-15"	);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) PMU_FORMAT_ATTR(edge,	"config:18"	);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) PMU_FORMAT_ATTR(inv,	"config:23"	);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) PMU_FORMAT_ATTR(cmask,	"config:24-31"	);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static struct attribute *intel_knc_formats_attr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	&format_attr_event.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	&format_attr_umask.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	&format_attr_edge.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	&format_attr_inv.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	&format_attr_cmask.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static const struct x86_pmu knc_pmu __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	.name			= "knc",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	.handle_irq		= knc_pmu_handle_irq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	.disable_all		= knc_pmu_disable_all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	.enable_all		= knc_pmu_enable_all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	.enable			= knc_pmu_enable_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	.disable		= knc_pmu_disable_event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	.hw_config		= x86_pmu_hw_config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	.schedule_events	= x86_schedule_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	.eventsel		= MSR_KNC_EVNTSEL0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	.perfctr		= MSR_KNC_PERFCTR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	.event_map		= knc_pmu_event_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	.max_events             = ARRAY_SIZE(knc_perfmon_event_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	.apic			= 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	.max_period		= (1ULL << 39) - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	.version		= 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	.num_counters		= 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	.cntval_bits		= 40,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	.cntval_mask		= (1ULL << 40) - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	.get_event_constraints	= x86_get_event_constraints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	.event_constraints	= knc_event_constraints,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	.format_attrs		= intel_knc_formats_attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) __init int knc_pmu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	x86_pmu = knc_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	memcpy(hw_cache_event_ids, knc_hw_cache_event_ids, 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 		sizeof(hw_cache_event_ids));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }