^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Intel(R) Processor Trace PMU driver for perf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2013-2014, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Intel PT is specified in the Intel Architecture Instruction Set Extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Programming Reference:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * http://software.intel.com/en-us/intel-isa-extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #undef DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/intel_pt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/intel-family.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "../perf_event.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include "pt.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static DEFINE_PER_CPU(struct pt, pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static struct pt_pmu pt_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Capabilities of Intel PT hardware, such as number of address bits or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * supported output schemes, are cached and exported to userspace as "caps"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * attribute group of pt pmu device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * (/sys/bus/event_source/devices/intel_pt/caps/) so that userspace can store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * relevant bits together with intel_pt traces.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * These are necessary for both trace decoding (payloads_lip, contains address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * width encoded in IP-related packets), and event configuration (bitmasks with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * permitted values for certain bit fields).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define PT_CAP(_n, _l, _r, _m) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) [PT_CAP_ ## _n] = { .name = __stringify(_n), .leaf = _l, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) .reg = _r, .mask = _m }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static struct pt_cap_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u32 leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) u8 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) } pt_caps[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) PT_CAP(max_subleaf, 0, CPUID_EAX, 0xffffffff),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) PT_CAP(cr3_filtering, 0, CPUID_EBX, BIT(0)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) PT_CAP(psb_cyc, 0, CPUID_EBX, BIT(1)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) PT_CAP(ip_filtering, 0, CPUID_EBX, BIT(2)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) PT_CAP(mtc, 0, CPUID_EBX, BIT(3)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) PT_CAP(ptwrite, 0, CPUID_EBX, BIT(4)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) PT_CAP(power_event_trace, 0, CPUID_EBX, BIT(5)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) PT_CAP(topa_output, 0, CPUID_ECX, BIT(0)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) PT_CAP(topa_multiple_entries, 0, CPUID_ECX, BIT(1)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) PT_CAP(single_range_output, 0, CPUID_ECX, BIT(2)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) PT_CAP(output_subsys, 0, CPUID_ECX, BIT(3)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) PT_CAP(payloads_lip, 0, CPUID_ECX, BIT(31)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) PT_CAP(num_address_ranges, 1, CPUID_EAX, 0x7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) PT_CAP(mtc_periods, 1, CPUID_EAX, 0xffff0000),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) PT_CAP(cycle_thresholds, 1, CPUID_EBX, 0xffff),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) PT_CAP(psb_periods, 1, CPUID_EBX, 0xffff0000),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u32 intel_pt_validate_cap(u32 *caps, enum pt_capabilities capability)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct pt_cap_desc *cd = &pt_caps[capability];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) u32 c = caps[cd->leaf * PT_CPUID_REGS_NUM + cd->reg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) unsigned int shift = __ffs(cd->mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) return (c & cd->mask) >> shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) EXPORT_SYMBOL_GPL(intel_pt_validate_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u32 intel_pt_validate_hw_cap(enum pt_capabilities cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return intel_pt_validate_cap(pt_pmu.caps, cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) EXPORT_SYMBOL_GPL(intel_pt_validate_hw_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static ssize_t pt_cap_show(struct device *cdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct dev_ext_attribute *ea =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) container_of(attr, struct dev_ext_attribute, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) enum pt_capabilities cap = (long)ea->var;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return snprintf(buf, PAGE_SIZE, "%x\n", intel_pt_validate_hw_cap(cap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static struct attribute_group pt_cap_group __ro_after_init = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) .name = "caps",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) PMU_FORMAT_ATTR(pt, "config:0" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) PMU_FORMAT_ATTR(cyc, "config:1" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) PMU_FORMAT_ATTR(pwr_evt, "config:4" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) PMU_FORMAT_ATTR(fup_on_ptw, "config:5" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) PMU_FORMAT_ATTR(mtc, "config:9" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) PMU_FORMAT_ATTR(tsc, "config:10" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) PMU_FORMAT_ATTR(noretcomp, "config:11" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) PMU_FORMAT_ATTR(ptw, "config:12" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) PMU_FORMAT_ATTR(branch, "config:13" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) PMU_FORMAT_ATTR(mtc_period, "config:14-17" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) PMU_FORMAT_ATTR(cyc_thresh, "config:19-22" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) PMU_FORMAT_ATTR(psb_period, "config:24-27" );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static struct attribute *pt_formats_attr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) &format_attr_pt.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) &format_attr_cyc.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) &format_attr_pwr_evt.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) &format_attr_fup_on_ptw.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) &format_attr_mtc.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) &format_attr_tsc.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) &format_attr_noretcomp.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) &format_attr_ptw.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) &format_attr_branch.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) &format_attr_mtc_period.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) &format_attr_cyc_thresh.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) &format_attr_psb_period.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static struct attribute_group pt_format_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) .name = "format",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) .attrs = pt_formats_attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) pt_timing_attr_show(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) char *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct perf_pmu_events_attr *pmu_attr =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) container_of(attr, struct perf_pmu_events_attr, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) switch (pmu_attr->id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return sprintf(page, "%lu\n", pt_pmu.max_nonturbo_ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return sprintf(page, "%u:%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) pt_pmu.tsc_art_num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pt_pmu.tsc_art_den);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) PMU_EVENT_ATTR(max_nonturbo_ratio, timing_attr_max_nonturbo_ratio, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) pt_timing_attr_show);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) PMU_EVENT_ATTR(tsc_art_ratio, timing_attr_tsc_art_ratio, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) pt_timing_attr_show);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static struct attribute *pt_timing_attr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) &timing_attr_max_nonturbo_ratio.attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) &timing_attr_tsc_art_ratio.attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static struct attribute_group pt_timing_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) .attrs = pt_timing_attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static const struct attribute_group *pt_attr_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) &pt_cap_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) &pt_format_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) &pt_timing_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static int __init pt_pmu_hw_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) struct dev_ext_attribute *de_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct attribute **attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) u64 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) rdmsrl(MSR_PLATFORM_INFO, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) pt_pmu.max_nonturbo_ratio = (reg & 0xff00) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * if available, read in TSC to core crystal clock ratio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * otherwise, zero for numerator stands for "not enumerated"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * as per SDM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (boot_cpu_data.cpuid_level >= CPUID_TSC_LEAF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) u32 eax, ebx, ecx, edx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) cpuid(CPUID_TSC_LEAF, &eax, &ebx, &ecx, &edx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) pt_pmu.tsc_art_num = ebx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) pt_pmu.tsc_art_den = eax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /* model-specific quirks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) switch (boot_cpu_data.x86_model) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) case INTEL_FAM6_BROADWELL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) case INTEL_FAM6_BROADWELL_D:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) case INTEL_FAM6_BROADWELL_G:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) case INTEL_FAM6_BROADWELL_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* not setting BRANCH_EN will #GP, erratum BDM106 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) pt_pmu.branch_en_always_on = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (boot_cpu_has(X86_FEATURE_VMX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * Intel SDM, 36.5 "Tracing post-VMXON" says that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * "IA32_VMX_MISC[bit 14]" being 1 means PT can trace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * post-VMXON.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) rdmsrl(MSR_IA32_VMX_MISC, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (reg & BIT(14))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) pt_pmu.vmx = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) for (i = 0; i < PT_CPUID_LEAVES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) cpuid_count(20, i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) &pt_pmu.caps[CPUID_EAX + i*PT_CPUID_REGS_NUM],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) &pt_pmu.caps[CPUID_EBX + i*PT_CPUID_REGS_NUM],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) &pt_pmu.caps[CPUID_ECX + i*PT_CPUID_REGS_NUM],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) &pt_pmu.caps[CPUID_EDX + i*PT_CPUID_REGS_NUM]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) size = sizeof(struct attribute *) * (ARRAY_SIZE(pt_caps)+1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) attrs = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (!attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) size = sizeof(struct dev_ext_attribute) * (ARRAY_SIZE(pt_caps)+1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) de_attrs = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!de_attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) for (i = 0; i < ARRAY_SIZE(pt_caps); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct dev_ext_attribute *de_attr = de_attrs + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) de_attr->attr.attr.name = pt_caps[i].name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) sysfs_attr_init(&de_attr->attr.attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) de_attr->attr.attr.mode = S_IRUGO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) de_attr->attr.show = pt_cap_show;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) de_attr->var = (void *)i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) attrs[i] = &de_attr->attr.attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) pt_cap_group.attrs = attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) kfree(attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #define RTIT_CTL_CYC_PSB (RTIT_CTL_CYCLEACC | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) RTIT_CTL_CYC_THRESH | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) RTIT_CTL_PSB_FREQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #define RTIT_CTL_MTC (RTIT_CTL_MTC_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) RTIT_CTL_MTC_RANGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #define RTIT_CTL_PTW (RTIT_CTL_PTW_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) RTIT_CTL_FUP_ON_PTW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * Bit 0 (TraceEn) in the attr.config is meaningless as the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * corresponding bit in the RTIT_CTL can only be controlled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * by the driver; therefore, repurpose it to mean: pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * through the bit that was previously assumed to be always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * on for PT, thereby allowing the user to *not* set it if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * they so wish. See also pt_event_valid() and pt_config().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #define RTIT_CTL_PASSTHROUGH RTIT_CTL_TRACEEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #define PT_CONFIG_MASK (RTIT_CTL_TRACEEN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) RTIT_CTL_TSC_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) RTIT_CTL_DISRETC | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) RTIT_CTL_BRANCH_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) RTIT_CTL_CYC_PSB | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) RTIT_CTL_MTC | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) RTIT_CTL_PWR_EVT_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) RTIT_CTL_FUP_ON_PTW | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) RTIT_CTL_PTW_EN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static bool pt_event_valid(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) u64 config = event->attr.config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) u64 allowed, requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if ((config & PT_CONFIG_MASK) != config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (config & RTIT_CTL_CYC_PSB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!intel_pt_validate_hw_cap(PT_CAP_psb_cyc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) allowed = intel_pt_validate_hw_cap(PT_CAP_psb_periods);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) requested = (config & RTIT_CTL_PSB_FREQ) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) RTIT_CTL_PSB_FREQ_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (requested && (!(allowed & BIT(requested))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) allowed = intel_pt_validate_hw_cap(PT_CAP_cycle_thresholds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) requested = (config & RTIT_CTL_CYC_THRESH) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) RTIT_CTL_CYC_THRESH_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (requested && (!(allowed & BIT(requested))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (config & RTIT_CTL_MTC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * In the unlikely case that CPUID lists valid mtc periods,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * but not the mtc capability, drop out here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * Spec says that setting mtc period bits while mtc bit in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * CPUID is 0 will #GP, so better safe than sorry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (!intel_pt_validate_hw_cap(PT_CAP_mtc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) allowed = intel_pt_validate_hw_cap(PT_CAP_mtc_periods);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (!allowed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) requested = (config & RTIT_CTL_MTC_RANGE) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) RTIT_CTL_MTC_RANGE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (!(allowed & BIT(requested)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (config & RTIT_CTL_PWR_EVT_EN &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) !intel_pt_validate_hw_cap(PT_CAP_power_event_trace))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (config & RTIT_CTL_PTW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (!intel_pt_validate_hw_cap(PT_CAP_ptwrite))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /* FUPonPTW without PTW doesn't make sense */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if ((config & RTIT_CTL_FUP_ON_PTW) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) !(config & RTIT_CTL_PTW_EN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * Setting bit 0 (TraceEn in RTIT_CTL MSR) in the attr.config
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * clears the assomption that BranchEn must always be enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * as was the case with the first implementation of PT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * If this bit is not set, the legacy behavior is preserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * for compatibility with the older userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * Re-using bit 0 for this purpose is fine because it is never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * directly set by the user; previous attempts at setting it in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * the attr.config resulted in -EINVAL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (config & RTIT_CTL_PASSTHROUGH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * Disallow not setting BRANCH_EN where BRANCH_EN is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * always required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (pt_pmu.branch_en_always_on &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) !(config & RTIT_CTL_BRANCH_EN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * Disallow BRANCH_EN without the PASSTHROUGH.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (config & RTIT_CTL_BRANCH_EN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * PT configuration helpers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * These all are cpu affine and operate on a local PT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static void pt_config_start(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) u64 ctl = event->hw.config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) ctl |= RTIT_CTL_TRACEEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (READ_ONCE(pt->vmx_on))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) wrmsrl(MSR_IA32_RTIT_CTL, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) WRITE_ONCE(event->hw.config, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* Address ranges and their corresponding msr configuration registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static const struct pt_address_range {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) unsigned long msr_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) unsigned long msr_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) unsigned int reg_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) } pt_address_ranges[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) .msr_a = MSR_IA32_RTIT_ADDR0_A,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) .msr_b = MSR_IA32_RTIT_ADDR0_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) .reg_off = RTIT_CTL_ADDR0_OFFSET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) .msr_a = MSR_IA32_RTIT_ADDR1_A,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) .msr_b = MSR_IA32_RTIT_ADDR1_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) .reg_off = RTIT_CTL_ADDR1_OFFSET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) .msr_a = MSR_IA32_RTIT_ADDR2_A,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) .msr_b = MSR_IA32_RTIT_ADDR2_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) .reg_off = RTIT_CTL_ADDR2_OFFSET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) .msr_a = MSR_IA32_RTIT_ADDR3_A,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) .msr_b = MSR_IA32_RTIT_ADDR3_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) .reg_off = RTIT_CTL_ADDR3_OFFSET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static u64 pt_config_filters(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct pt_filters *filters = event->hw.addr_filters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned int range = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) u64 rtit_ctl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (!filters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) perf_event_addr_filters_sync(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) for (range = 0; range < filters->nr_filters; range++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct pt_filter *filter = &filters->filter[range];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * Note, if the range has zero start/end addresses due
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * to its dynamic object not being loaded yet, we just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * go ahead and program zeroed range, which will simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * produce no data. Note^2: if executable code at 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * is a concern, we can set up an "invalid" configuration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * such as msr_b < msr_a.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* avoid redundant msr writes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (pt->filters.filter[range].msr_a != filter->msr_a) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) wrmsrl(pt_address_ranges[range].msr_a, filter->msr_a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) pt->filters.filter[range].msr_a = filter->msr_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (pt->filters.filter[range].msr_b != filter->msr_b) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) wrmsrl(pt_address_ranges[range].msr_b, filter->msr_b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) pt->filters.filter[range].msr_b = filter->msr_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) rtit_ctl |= (u64)filter->config << pt_address_ranges[range].reg_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return rtit_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static void pt_config(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct pt_buffer *buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) u64 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /* First round: clear STATUS, in particular the PSB byte counter. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (!event->hw.config) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) perf_event_itrace_started(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) wrmsrl(MSR_IA32_RTIT_STATUS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) reg = pt_config_filters(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) reg |= RTIT_CTL_TRACEEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!buf->single)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) reg |= RTIT_CTL_TOPA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Previously, we had BRANCH_EN on by default, but now that PT has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * grown features outside of branch tracing, it is useful to allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * the user to disable it. Setting bit 0 in the event's attr.config
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * allows BRANCH_EN to pass through instead of being always on. See
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * also the comment in pt_event_valid().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (event->attr.config & BIT(0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) reg |= event->attr.config & RTIT_CTL_BRANCH_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) reg |= RTIT_CTL_BRANCH_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (!event->attr.exclude_kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) reg |= RTIT_CTL_OS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!event->attr.exclude_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) reg |= RTIT_CTL_USR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) reg |= (event->attr.config & PT_CONFIG_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) event->hw.config = reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) pt_config_start(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) static void pt_config_stop(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) u64 ctl = READ_ONCE(event->hw.config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* may be already stopped by a PMI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (!(ctl & RTIT_CTL_TRACEEN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) ctl &= ~RTIT_CTL_TRACEEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (!READ_ONCE(pt->vmx_on))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) wrmsrl(MSR_IA32_RTIT_CTL, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) WRITE_ONCE(event->hw.config, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * A wrmsr that disables trace generation serializes other PT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * registers and causes all data packets to be written to memory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * but a fence is required for the data to become globally visible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * The below WMB, separating data store and aux_head store matches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * the consumer's RMB that separates aux_head load and data load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * struct topa - ToPA metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * @list: linkage to struct pt_buffer's list of tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * @offset: offset of the first entry in this table in the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * @size: total size of all entries in this table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * @last: index of the last initialized entry in this table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * @z_count: how many times the first entry repeats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct topa {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) u64 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) int last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned int z_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * Keep ToPA table-related metadata on the same page as the actual table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * taking up a few words from the top
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) #define TENTS_PER_PAGE \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) ((PAGE_SIZE - sizeof(struct topa)) / sizeof(struct topa_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * struct topa_page - page-sized ToPA table with metadata at the top
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * @table: actual ToPA table entries, as understood by PT hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * @topa: metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct topa_page {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) struct topa_entry table[TENTS_PER_PAGE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct topa topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) static inline struct topa_page *topa_to_page(struct topa *topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return container_of(topa, struct topa_page, topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) static inline struct topa_page *topa_entry_to_page(struct topa_entry *te)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) return (struct topa_page *)((unsigned long)te & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) static inline phys_addr_t topa_pfn(struct topa *topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return PFN_DOWN(virt_to_phys(topa_to_page(topa)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /* make -1 stand for the last table entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #define TOPA_ENTRY(t, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) ((i) == -1 \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) ? &topa_to_page(t)->table[(t)->last] \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) : &topa_to_page(t)->table[(i)])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) #define TOPA_ENTRY_SIZE(t, i) (sizes(TOPA_ENTRY((t), (i))->size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) #define TOPA_ENTRY_PAGES(t, i) (1 << TOPA_ENTRY((t), (i))->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static void pt_config_buffer(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u64 reg, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) void *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (buf->single) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) base = buf->data_pages[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) mask = (buf->nr_pages * PAGE_SIZE - 1) >> 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) base = topa_to_page(buf->cur)->table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) mask = (u64)buf->cur_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) reg = virt_to_phys(base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (pt->output_base != reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) pt->output_base = reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) reg = 0x7f | (mask << 7) | ((u64)buf->output_off << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (pt->output_mask != reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) pt->output_mask = reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * topa_alloc() - allocate page-sized ToPA table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * @cpu: CPU on which to allocate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * @gfp: Allocation flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * Return: On success, return the pointer to ToPA table page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) static struct topa *topa_alloc(int cpu, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) int node = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) struct topa_page *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) p = alloc_pages_node(node, gfp | __GFP_ZERO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) tp = page_address(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) tp->topa.last = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * In case of singe-entry ToPA, always put the self-referencing END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * link as the 2nd entry in the table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) TOPA_ENTRY(&tp->topa, 1)->base = page_to_phys(p) >> TOPA_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) TOPA_ENTRY(&tp->topa, 1)->end = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return &tp->topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * topa_free() - free a page-sized ToPA table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * @topa: Table to deallocate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) static void topa_free(struct topa *topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) free_page((unsigned long)topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * topa_insert_table() - insert a ToPA table into a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * @buf: PT buffer that's being extended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * @topa: New topa table to be inserted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * If it's the first table in this buffer, set up buffer's pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * accordingly; otherwise, add a END=1 link entry to @topa to the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * "last" table and adjust the last table pointer to @topa.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static void topa_insert_table(struct pt_buffer *buf, struct topa *topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) struct topa *last = buf->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) list_add_tail(&topa->list, &buf->tables);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if (!buf->first) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) buf->first = buf->last = buf->cur = topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) topa->offset = last->offset + last->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) buf->last = topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) BUG_ON(last->last != TENTS_PER_PAGE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) TOPA_ENTRY(last, -1)->base = topa_pfn(topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) TOPA_ENTRY(last, -1)->end = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * topa_table_full() - check if a ToPA table is filled up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * @topa: ToPA table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) static bool topa_table_full(struct topa *topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) /* single-entry ToPA is a special case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return !!topa->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return topa->last == TENTS_PER_PAGE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * topa_insert_pages() - create a list of ToPA tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * @buf: PT buffer being initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * @gfp: Allocation flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * This initializes a list of ToPA tables with entries from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * the data_pages provided by rb_alloc_aux().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * Return: 0 on success or error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) static int topa_insert_pages(struct pt_buffer *buf, int cpu, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) struct topa *topa = buf->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) int order = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) p = virt_to_page(buf->data_pages[buf->nr_pages]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (PagePrivate(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) order = page_private(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (topa_table_full(topa)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) topa = topa_alloc(cpu, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (!topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) topa_insert_table(buf, topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (topa->z_count == topa->last - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (order == TOPA_ENTRY(topa, topa->last - 1)->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) topa->z_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) TOPA_ENTRY(topa, -1)->base = page_to_phys(p) >> TOPA_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) TOPA_ENTRY(topa, -1)->size = order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (!buf->snapshot &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) !intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) TOPA_ENTRY(topa, -1)->intr = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) TOPA_ENTRY(topa, -1)->stop = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) topa->last++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) topa->size += sizes(order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) buf->nr_pages += 1ul << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * pt_topa_dump() - print ToPA tables and their entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) static void pt_topa_dump(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) struct topa *topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) list_for_each_entry(topa, &buf->tables, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) struct topa_page *tp = topa_to_page(topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) pr_debug("# table @%p, off %llx size %zx\n", tp->table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) topa->offset, topa->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) for (i = 0; i < TENTS_PER_PAGE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) pr_debug("# entry @%p (%lx sz %u %c%c%c) raw=%16llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) &tp->table[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) (unsigned long)tp->table[i].base << TOPA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) sizes(tp->table[i].size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) tp->table[i].end ? 'E' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) tp->table[i].intr ? 'I' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) tp->table[i].stop ? 'S' : ' ',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) *(u64 *)&tp->table[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if ((intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) tp->table[i].stop) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) tp->table[i].end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (!i && topa->z_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) i += topa->z_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * pt_buffer_advance() - advance to the next output region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * Advance the current pointers in the buffer to the next ToPA entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static void pt_buffer_advance(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) buf->output_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) buf->cur_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (buf->cur_idx == buf->cur->last) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (buf->cur == buf->last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) buf->cur = buf->first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) buf->cur = list_entry(buf->cur->list.next, struct topa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) buf->cur_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * pt_update_head() - calculate current offsets and sizes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * @pt: Per-cpu pt context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * Update buffer's current write pointer position and data size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) static void pt_update_head(struct pt *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) struct pt_buffer *buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) u64 topa_idx, base, old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (buf->single) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) local_set(&buf->data_size, buf->output_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) /* offset of the first region in this table from the beginning of buf */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) base = buf->cur->offset + buf->output_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) /* offset of the current output region within this table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) for (topa_idx = 0; topa_idx < buf->cur_idx; topa_idx++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) base += TOPA_ENTRY_SIZE(buf->cur, topa_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (buf->snapshot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) local_set(&buf->data_size, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) old = (local64_xchg(&buf->head, base) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) ((buf->nr_pages << PAGE_SHIFT) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (base < old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) base += buf->nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) local_add(base - old, &buf->data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * pt_buffer_region() - obtain current output region's address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) static void *pt_buffer_region(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) return phys_to_virt(TOPA_ENTRY(buf->cur, buf->cur_idx)->base << TOPA_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * pt_buffer_region_size() - obtain current output region's size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) static size_t pt_buffer_region_size(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) return TOPA_ENTRY_SIZE(buf->cur, buf->cur_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * pt_handle_status() - take care of possible status conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * @pt: Per-cpu pt context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) static void pt_handle_status(struct pt *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct pt_buffer *buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) int advance = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) u64 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) rdmsrl(MSR_IA32_RTIT_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (status & RTIT_STATUS_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) pr_err_ratelimited("ToPA ERROR encountered, trying to recover\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) pt_topa_dump(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) status &= ~RTIT_STATUS_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (status & RTIT_STATUS_STOPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) status &= ~RTIT_STATUS_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * On systems that only do single-entry ToPA, hitting STOP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * means we are already losing data; need to let the decoder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * know.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) if (!buf->single &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) buf->output_off == pt_buffer_region_size(buf))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) perf_aux_output_flag(&pt->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) PERF_AUX_FLAG_TRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) advance++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * Also on single-entry ToPA implementations, interrupt will come
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * before the output reaches its output region's boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) !buf->snapshot &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) pt_buffer_region_size(buf) - buf->output_off <= TOPA_PMI_MARGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) void *head = pt_buffer_region(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) /* everything within this margin needs to be zeroed out */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) memset(head + buf->output_off, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) pt_buffer_region_size(buf) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) buf->output_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) advance++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) if (advance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) pt_buffer_advance(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) wrmsrl(MSR_IA32_RTIT_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * pt_read_offset() - translate registers into buffer pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * Set buffer's output pointers from MSR values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) static void pt_read_offset(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) struct topa_page *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) if (!buf->single) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) tp = phys_to_virt(pt->output_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) buf->cur = &tp->topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) /* offset within current output region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) buf->output_off = pt->output_mask >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /* index of current output region within this table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (!buf->single)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) buf->cur_idx = (pt->output_mask & 0xffffff80) >> 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) static struct topa_entry *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) pt_topa_entry_for_page(struct pt_buffer *buf, unsigned int pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) struct topa_page *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) struct topa *topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) unsigned int idx, cur_pg = 0, z_pg = 0, start_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * Indicates a bug in the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (WARN_ON_ONCE(pg >= buf->nr_pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * First, find the ToPA table where @pg fits. With high
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * order allocations, there shouldn't be many of these.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) list_for_each_entry(topa, &buf->tables, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (topa->offset + topa->size > pg << PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * Hitting this means we have a problem in the ToPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * allocation code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) * Indicates a problem in the ToPA allocation code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) if (WARN_ON_ONCE(topa->last == -1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) tp = topa_to_page(topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) cur_pg = PFN_DOWN(topa->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (topa->z_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) z_pg = TOPA_ENTRY_PAGES(topa, 0) * (topa->z_count + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) start_idx = topa->z_count + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * Multiple entries at the beginning of the table have the same size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) * ideally all of them; if @pg falls there, the search is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) if (pg >= cur_pg && pg < cur_pg + z_pg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) idx = (pg - cur_pg) / TOPA_ENTRY_PAGES(topa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) return &tp->table[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * Otherwise, slow path: iterate through the remaining entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) for (idx = start_idx, cur_pg += z_pg; idx < topa->last; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (cur_pg + TOPA_ENTRY_PAGES(topa, idx) > pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) return &tp->table[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) cur_pg += TOPA_ENTRY_PAGES(topa, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * Means we couldn't find a ToPA entry in the table that does match.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) static struct topa_entry *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) pt_topa_prev_entry(struct pt_buffer *buf, struct topa_entry *te)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) unsigned long table = (unsigned long)te & ~(PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) struct topa_page *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) struct topa *topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) tp = (struct topa_page *)table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (tp->table != te)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) return --te;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) topa = &tp->topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) if (topa == buf->first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) topa = buf->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) topa = list_prev_entry(topa, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) tp = topa_to_page(topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) return &tp->table[topa->last - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) * pt_buffer_reset_markers() - place interrupt and stop bits in the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * @handle: Current output handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * Place INT and STOP marks to prevent overwriting old data that the consumer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * hasn't yet collected and waking up the consumer after a certain fraction of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) * the buffer has filled up. Only needed and sensible for non-snapshot counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * This obviously relies on buf::head to figure out buffer markers, so it has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * to be called after pt_buffer_reset_offsets() and before the hardware tracing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) * is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) static int pt_buffer_reset_markers(struct pt_buffer *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) unsigned long head = local64_read(&buf->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) unsigned long idx, npages, wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (buf->single)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /* can't stop in the middle of an output region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) if (buf->output_off + handle->size + 1 < pt_buffer_region_size(buf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) /* single entry ToPA is handled by marking all regions STOP=1 INT=1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /* clear STOP and INT from current entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) if (buf->stop_te) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) buf->stop_te->stop = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) buf->stop_te->intr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) if (buf->intr_te)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) buf->intr_te->intr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) /* how many pages till the STOP marker */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) npages = handle->size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) /* if it's on a page boundary, fill up one more page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) if (!offset_in_page(head + handle->size + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) npages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) idx = (head >> PAGE_SHIFT) + npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) idx &= buf->nr_pages - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (idx != buf->stop_pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) buf->stop_pos = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) buf->stop_te = pt_topa_entry_for_page(buf, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) buf->stop_te = pt_topa_prev_entry(buf, buf->stop_te);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) wakeup = handle->wakeup >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) /* in the worst case, wake up the consumer one page before hard stop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) idx = (head >> PAGE_SHIFT) + npages - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (idx > wakeup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) idx = wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) idx &= buf->nr_pages - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (idx != buf->intr_pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) buf->intr_pos = idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) buf->intr_te = pt_topa_entry_for_page(buf, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) buf->intr_te = pt_topa_prev_entry(buf, buf->intr_te);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) buf->stop_te->stop = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) buf->stop_te->intr = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) buf->intr_te->intr = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) * pt_buffer_reset_offsets() - adjust buffer's write pointers from aux_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) * @head: Write pointer (aux_head) from AUX buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) * Find the ToPA table and entry corresponding to given @head and set buffer's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * "current" pointers accordingly. This is done after we have obtained the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * current aux_head position from a successful call to perf_aux_output_begin()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * to make sure the hardware is writing to the right place.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) * This function modifies buf::{cur,cur_idx,output_off} that will be programmed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * into PT msrs when the tracing is enabled and buf::head and buf::data_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * which are used to determine INT and STOP markers' locations by a subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * call to pt_buffer_reset_markers().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) static void pt_buffer_reset_offsets(struct pt_buffer *buf, unsigned long head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) struct topa_page *cur_tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) struct topa_entry *te;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) int pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) if (buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) head &= (buf->nr_pages << PAGE_SHIFT) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) if (!buf->single) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) pg = (head >> PAGE_SHIFT) & (buf->nr_pages - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) te = pt_topa_entry_for_page(buf, pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) cur_tp = topa_entry_to_page(te);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) buf->cur = &cur_tp->topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) buf->cur_idx = te - TOPA_ENTRY(buf->cur, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) buf->output_off = head & (pt_buffer_region_size(buf) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) buf->output_off = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) local64_set(&buf->head, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) local_set(&buf->data_size, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * pt_buffer_fini_topa() - deallocate ToPA structure of a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) static void pt_buffer_fini_topa(struct pt_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct topa *topa, *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (buf->single)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) list_for_each_entry_safe(topa, iter, &buf->tables, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * right now, this is in free_aux() path only, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * no need to unlink this table from the list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) topa_free(topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * pt_buffer_init_topa() - initialize ToPA table for pt buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) * @buf: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) * @size: Total size of all regions within this ToPA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) * @gfp: Allocation flags.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) static int pt_buffer_init_topa(struct pt_buffer *buf, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) unsigned long nr_pages, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) struct topa *topa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) topa = topa_alloc(cpu, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) if (!topa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) topa_insert_table(buf, topa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) while (buf->nr_pages < nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) err = topa_insert_pages(buf, cpu, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) pt_buffer_fini_topa(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) /* link last table to the first one, unless we're double buffering */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) if (intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) TOPA_ENTRY(buf->last, -1)->base = topa_pfn(buf->first);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) TOPA_ENTRY(buf->last, -1)->end = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) pt_topa_dump(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) struct page *p = virt_to_page(buf->data_pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) int ret = -ENOTSUPP, order = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * We can use single range output mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * + in snapshot mode, where we don't need interrupts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) * + if the hardware supports it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * + if the entire buffer is one contiguous allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) if (!buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) if (!intel_pt_validate_hw_cap(PT_CAP_single_range_output))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) if (PagePrivate(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) order = page_private(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) if (1 << order != nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) buf->single = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) buf->nr_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * pt_buffer_setup_aux() - set up topa tables for a PT buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * @cpu: Cpu on which to allocate, -1 means current.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * @pages: Array of pointers to buffer pages passed from perf core.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) * @nr_pages: Number of pages in the buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * @snapshot: If this is a snapshot/overwrite counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * This is a pmu::setup_aux callback that sets up ToPA tables and all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * bookkeeping for an AUX buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * Return: Our private PT buffer structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) static void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) pt_buffer_setup_aux(struct perf_event *event, void **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) int nr_pages, bool snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) struct pt_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) int node, ret, cpu = event->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (!nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) * Only support AUX sampling in snapshot mode, where we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) * generate NMIs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (event->attr.aux_sample_size && !snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) if (cpu == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) node = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) buf = kzalloc_node(sizeof(struct pt_buffer), GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) buf->snapshot = snapshot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) buf->data_pages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) buf->stop_pos = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) buf->intr_pos = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) INIT_LIST_HEAD(&buf->tables);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) ret = pt_buffer_try_single(buf, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) ret = pt_buffer_init_topa(buf, cpu, nr_pages, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) * pt_buffer_free_aux() - perf AUX deallocation path callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) * @data: PT buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) static void pt_buffer_free_aux(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) struct pt_buffer *buf = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) pt_buffer_fini_topa(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) static int pt_addr_filters_init(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) struct pt_filters *filters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) int node = event->cpu == -1 ? -1 : cpu_to_node(event->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) if (!intel_pt_validate_hw_cap(PT_CAP_num_address_ranges))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) filters = kzalloc_node(sizeof(struct pt_filters), GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (!filters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) if (event->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) memcpy(filters, event->parent->hw.addr_filters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) sizeof(*filters));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) event->hw.addr_filters = filters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) static void pt_addr_filters_fini(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) kfree(event->hw.addr_filters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) event->hw.addr_filters = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) static inline bool valid_kernel_ip(unsigned long ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) return virt_addr_valid(ip) && kernel_ip(ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) static int pt_event_addr_filters_validate(struct list_head *filters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) struct perf_addr_filter *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) int range = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) list_for_each_entry(filter, filters, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * PT doesn't support single address triggers and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * 'start' filters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (!filter->size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) filter->action == PERF_ADDR_FILTER_ACTION_START)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) if (!filter->path.dentry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) if (!valid_kernel_ip(filter->offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) if (!valid_kernel_ip(filter->offset + filter->size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) if (++range > intel_pt_validate_hw_cap(PT_CAP_num_address_ranges))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) static void pt_event_addr_filters_sync(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) struct perf_addr_filters_head *head = perf_event_addr_filters(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) unsigned long msr_a, msr_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) struct perf_addr_filter_range *fr = event->addr_filter_ranges;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) struct pt_filters *filters = event->hw.addr_filters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) struct perf_addr_filter *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) int range = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) if (!filters)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) list_for_each_entry(filter, &head->list, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) if (filter->path.dentry && !fr[range].start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) msr_a = msr_b = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) /* apply the offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) msr_a = fr[range].start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) msr_b = msr_a + fr[range].size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) filters->filter[range].msr_a = msr_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) filters->filter[range].msr_b = msr_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) filters->filter[range].config = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) filters->filter[range].config = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) range++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) filters->nr_filters = range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * intel_pt_interrupt() - PT PMI handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) void intel_pt_interrupt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) struct pt_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct perf_event *event = pt->handle.event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * There may be a dangling PT bit in the interrupt status register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * after PT has been disabled by pt_event_stop(). Make sure we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * do anything (particularly, re-enable) for this event here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (!READ_ONCE(pt->handle_nmi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) pt_config_stop(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) pt_read_offset(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) pt_handle_status(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) pt_update_head(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) if (!event->hw.state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) buf = perf_aux_output_begin(&pt->handle, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) if (!buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) event->hw.state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) pt_buffer_reset_offsets(buf, pt->handle.head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) /* snapshot counters don't use PMI, so it's safe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) ret = pt_buffer_reset_markers(buf, &pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) perf_aux_output_end(&pt->handle, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) pt_config_buffer(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) pt_config_start(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) void intel_pt_handle_vmx(int on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) struct perf_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) /* PT plays nice with VMX, do nothing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) if (pt_pmu.vmx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) * VMXON will clear RTIT_CTL.TraceEn; we need to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) * sure to not try to set it while VMX is on. Disable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * interrupts to avoid racing with pmu callbacks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) * concurrent PMI should be handled fine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) WRITE_ONCE(pt->vmx_on, on);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) * If an AUX transaction is in progress, it will contain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) * gap(s), so flag it PARTIAL to inform the user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) event = pt->handle.event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) if (event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) perf_aux_output_flag(&pt->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) PERF_AUX_FLAG_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) /* Turn PTs back on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) if (!on && event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) wrmsrl(MSR_IA32_RTIT_CTL, event->hw.config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) EXPORT_SYMBOL_GPL(intel_pt_handle_vmx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) * PMU callbacks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) static void pt_event_start(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) struct pt_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) buf = perf_aux_output_begin(&pt->handle, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) goto fail_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) pt_buffer_reset_offsets(buf, pt->handle.head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) if (!buf->snapshot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) if (pt_buffer_reset_markers(buf, &pt->handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) goto fail_end_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) WRITE_ONCE(pt->handle_nmi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) hwc->state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) pt_config_buffer(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) pt_config(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) fail_end_stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) perf_aux_output_end(&pt->handle, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) fail_stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) hwc->state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) static void pt_event_stop(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * Protect against the PMI racing with disabling wrmsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * see comment in intel_pt_interrupt().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) WRITE_ONCE(pt->handle_nmi, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) pt_config_stop(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) if (event->hw.state == PERF_HES_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) event->hw.state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) if (mode & PERF_EF_UPDATE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) struct pt_buffer *buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) if (WARN_ON_ONCE(pt->handle.event != event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) pt_read_offset(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) pt_handle_status(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) pt_update_head(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) pt->handle.head =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) local_xchg(&buf->data_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) buf->nr_pages << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) static long pt_event_snapshot_aux(struct perf_event *event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) struct pt_buffer *buf = perf_get_aux(&pt->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) unsigned long from = 0, to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) if (WARN_ON_ONCE(!buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) * Sampling is only allowed on snapshot events;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) * see pt_buffer_setup_aux().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) if (WARN_ON_ONCE(!buf->snapshot))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * Here, handle_nmi tells us if the tracing is on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) if (READ_ONCE(pt->handle_nmi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) pt_config_stop(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) pt_read_offset(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) pt_update_head(pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) to = local_read(&buf->data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) if (to < size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) from = buf->nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) from += to - size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) ret = perf_output_copy_aux(&pt->handle, handle, from, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) * If the tracing was on when we turned up, restart it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * Compiler barrier not needed as we couldn't have been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) * preempted by anything that touches pt->handle_nmi.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (pt->handle_nmi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) pt_config_start(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) static void pt_event_del(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) pt_event_stop(event, PERF_EF_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) static int pt_event_add(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) int ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) if (pt->handle.event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) if (mode & PERF_EF_START) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) pt_event_start(event, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) if (hwc->state == PERF_HES_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) hwc->state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) static void pt_event_read(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) static void pt_event_destroy(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) pt_addr_filters_fini(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) x86_del_exclusive(x86_lbr_exclusive_pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) static int pt_event_init(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) if (event->attr.type != pt_pmu.pmu.type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (!pt_event_valid(event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) if (x86_add_exclusive(x86_lbr_exclusive_pt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) if (pt_addr_filters_init(event)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) x86_del_exclusive(x86_lbr_exclusive_pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) event->destroy = pt_event_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) void cpu_emergency_stop_pt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) struct pt *pt = this_cpu_ptr(&pt_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if (pt->handle.event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) pt_event_stop(pt->handle.event, PERF_EF_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) int is_intel_pt_event(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) return event->pmu == &pt_pmu.pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) static __init int pt_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) int ret, cpu, prior_warn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) BUILD_BUG_ON(sizeof(struct topa) > PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) if (!boot_cpu_has(X86_FEATURE_INTEL_PT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) u64 ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) ret = rdmsrl_safe_on_cpu(cpu, MSR_IA32_RTIT_CTL, &ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) if (!ret && (ctl & RTIT_CTL_TRACEEN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) prior_warn++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) if (prior_warn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) x86_add_exclusive(x86_lbr_exclusive_pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) pr_warn("PT is enabled at boot time, doing nothing\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) ret = pt_pmu_hw_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) if (!intel_pt_validate_hw_cap(PT_CAP_topa_output)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) pr_warn("ToPA output is not supported on this CPU\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) if (!intel_pt_validate_hw_cap(PT_CAP_topa_multiple_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) pt_pmu.pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) pt_pmu.pmu.capabilities |= PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) pt_pmu.pmu.attr_groups = pt_attr_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) pt_pmu.pmu.task_ctx_nr = perf_sw_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) pt_pmu.pmu.event_init = pt_event_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) pt_pmu.pmu.add = pt_event_add;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) pt_pmu.pmu.del = pt_event_del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) pt_pmu.pmu.start = pt_event_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) pt_pmu.pmu.stop = pt_event_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) pt_pmu.pmu.snapshot_aux = pt_event_snapshot_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) pt_pmu.pmu.read = pt_event_read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) pt_pmu.pmu.setup_aux = pt_buffer_setup_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) pt_pmu.pmu.free_aux = pt_buffer_free_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) pt_pmu.pmu.addr_filters_sync = pt_event_addr_filters_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) pt_pmu.pmu.addr_filters_validate = pt_event_addr_filters_validate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) pt_pmu.pmu.nr_addr_filters =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) intel_pt_validate_hw_cap(PT_CAP_num_address_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) ret = perf_pmu_register(&pt_pmu.pmu, "intel_pt", -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) arch_initcall(pt_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags