^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * BTS PMU driver for perf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2013-2014, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #undef DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/coredump.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "../perf_event.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct bts_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct perf_output_handle handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct debug_store ds_back;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) int state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* BTS context states: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /* no ongoing AUX transactions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) BTS_STATE_STOPPED = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* AUX transaction is on, BTS tracing is disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) BTS_STATE_INACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* AUX transaction is on, BTS tracing is running */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) BTS_STATE_ACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define BTS_RECORD_SIZE 24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define BTS_SAFETY_MARGIN 4080
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct bts_phys {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) unsigned long displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct bts_buffer {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) size_t real_size; /* multiple of BTS_RECORD_SIZE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) unsigned int nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned int nr_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) unsigned int cur_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) bool snapshot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) local_t data_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) local_t head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) unsigned long end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) void **data_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct bts_phys buf[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static struct pmu bts_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static int buf_nr_pages(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (!PagePrivate(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return 1 << page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static size_t buf_size(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return buf_nr_pages(page) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) bts_buffer_setup_aux(struct perf_event *event, void **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) int nr_pages, bool overwrite)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct bts_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int cpu = event->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) size_t size = nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) int pg, nbuf, pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* count all the high order buffers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) for (pg = 0, nbuf = 0; pg < nr_pages;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) page = virt_to_page(pages[pg]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) pg += buf_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) nbuf++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * to avoid interrupts in overwrite mode, only allow one physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (overwrite && nbuf > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) buf->nr_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) buf->nr_bufs = nbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) buf->snapshot = overwrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) buf->data_pages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) buf->real_size = size - size % BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) unsigned int __nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) page = virt_to_page(pages[pg]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __nr_pages = buf_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) buf->buf[nbuf].page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) buf->buf[nbuf].offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) buf->buf[nbuf].size -= pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) pg += __nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) offset += __nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static void bts_buffer_free_aux(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return buf->buf[idx].offset + buf->buf[idx].displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) bts_config_buffer(struct bts_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct bts_phys *phys = &buf->buf[buf->cur_buf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned long index, thresh = 0, end = phys->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct page *page = phys->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) index = local_read(&buf->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (!buf->snapshot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (buf->end < phys->offset + buf_size(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) end = buf->end - phys->offset - phys->displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) index -= phys->offset + phys->displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (end - index > BTS_SAFETY_MARGIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) thresh = end - BTS_SAFETY_MARGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) else if (end - index > BTS_RECORD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) thresh = end - BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) thresh = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ds->bts_index = ds->bts_buffer_base + index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ds->bts_absolute_maximum = ds->bts_buffer_base + end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) ds->bts_interrupt_threshold = !buf->snapshot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) ? ds->bts_buffer_base + thresh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) unsigned long index = head - phys->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) memset(page_address(phys->page) + index, 0, phys->size - index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static void bts_update(struct bts_ctx *bts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) int cpu = raw_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct bts_buffer *buf = perf_get_aux(&bts->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) head = index + bts_buffer_offset(buf, buf->cur_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) old = local_xchg(&buf->head, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (!buf->snapshot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (old == head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (ds->bts_index >= ds->bts_absolute_maximum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) perf_aux_output_flag(&bts->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) PERF_AUX_FLAG_TRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * old and head are always in the same physical buffer, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * can subtract them to get the data size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) local_add(head - old, &buf->data_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) local_set(&buf->data_size, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * Ordering PMU callbacks wrt themselves and the PMI is done by means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * of bts::state, which:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * - is set when bts::handle::event is valid, that is, between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * perf_aux_output_begin() and perf_aux_output_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * - is zero otherwise;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * - is ordered against bts::handle::event with a compiler barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static void __bts_event_start(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct bts_buffer *buf = perf_get_aux(&bts->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) u64 config = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (!buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) config |= ARCH_PERFMON_EVENTSEL_INT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (!event->attr.exclude_kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) config |= ARCH_PERFMON_EVENTSEL_OS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (!event->attr.exclude_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) config |= ARCH_PERFMON_EVENTSEL_USR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) bts_config_buffer(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * local barrier to make sure that ds configuration made it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * before we enable BTS and bts::state goes ACTIVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /* INACTIVE/STOPPED -> ACTIVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) intel_pmu_enable_bts(config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static void bts_event_start(struct perf_event *event, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct bts_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) buf = perf_aux_output_begin(&bts->handle, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) goto fail_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (bts_buffer_reset(buf, &bts->handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) goto fail_end_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) perf_event_itrace_started(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) event->hw.state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) __bts_event_start(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) fail_end_stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) perf_aux_output_end(&bts->handle, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) fail_stop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) event->hw.state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static void __bts_event_stop(struct perf_event *event, int state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) WRITE_ONCE(bts->state, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * No extra synchronization is mandated by the documentation to have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * BTS data stores globally visible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) intel_pmu_disable_bts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) static void bts_event_stop(struct perf_event *event, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct bts_buffer *buf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) int state = READ_ONCE(bts->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (state == BTS_STATE_ACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) __bts_event_stop(event, BTS_STATE_STOPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (state != BTS_STATE_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) buf = perf_get_aux(&bts->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) event->hw.state |= PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (flags & PERF_EF_UPDATE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) bts_update(bts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) bts->handle.head =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) local_xchg(&buf->data_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) buf->nr_pages << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) perf_aux_output_end(&bts->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) local_xchg(&buf->data_size, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) void intel_bts_enable_local(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) int state = READ_ONCE(bts->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * Here we transition from INACTIVE to ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * if we instead are STOPPED from the interrupt handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * stay that way. Can't be ACTIVE here though.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (state == BTS_STATE_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (bts->handle.event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) __bts_event_start(bts->handle.event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) void intel_bts_disable_local(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * Here we transition from ACTIVE to INACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * do nothing for STOPPED or INACTIVE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (bts->handle.event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) unsigned long head, space, next_space, pad, gap, skip, wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) unsigned int next_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct bts_phys *phys, *next_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) phys = &buf->buf[buf->cur_buf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) space = phys->offset + phys->displacement + phys->size - head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) pad = space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) if (space > handle->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) space = handle->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) space -= space % BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (space <= BTS_SAFETY_MARGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* See if next phys buffer has more space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) next_buf = buf->cur_buf + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (next_buf >= buf->nr_bufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) next_buf = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) next_phys = &buf->buf[next_buf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) gap = buf_size(phys->page) - phys->displacement - phys->size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) next_phys->displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) skip = pad + gap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (handle->size >= skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) next_space = next_phys->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (next_space + skip > handle->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) next_space = handle->size - skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) next_space -= next_space % BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (next_space > space || !space) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (pad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) bts_buffer_pad_out(phys, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) ret = perf_aux_output_skip(handle, skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /* Advance to next phys buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) phys = next_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) space = next_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) head = phys->offset + phys->displacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * After this, cur_buf and head won't match ds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * anymore, so we must not be racing with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * bts_update().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) buf->cur_buf = next_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) local_set(&buf->head, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* Don't go far beyond wakeup watermark */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) handle->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (space > wakeup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) space = wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) space -= space % BTS_RECORD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) buf->end = head + space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * If we have no space, the lost notification would have been sent when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * we hit absolute_maximum - see bts_update()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (!space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) int intel_bts_interrupt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct perf_event *event = bts->handle.event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct bts_buffer *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) s64 old_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) int err = -ENOSPC, handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * The only surefire way of knowing if this NMI is ours is by checking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * the write ptr against the PMI threshold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) handled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * so we can only be INACTIVE or STOPPED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) buf = perf_get_aux(&bts->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * Skip snapshot counters: they don't use the interrupt, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * there's no other way of telling, because the pointer will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * keep moving
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (buf->snapshot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) old_head = local_read(&buf->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) bts_update(bts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) /* no new data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (old_head == local_read(&buf->head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return handled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) buf = perf_aux_output_begin(&bts->handle, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) err = bts_buffer_reset(buf, &bts->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * BTS_STATE_STOPPED should be visible before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * cleared handle::event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) perf_aux_output_end(&bts->handle, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static void bts_event_del(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) bts_event_stop(event, PERF_EF_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static int bts_event_add(struct perf_event *event, int mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct hw_perf_event *hwc = &event->hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) event->hw.state = PERF_HES_STOPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (bts->handle.event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) if (mode & PERF_EF_START) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) bts_event_start(event, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (hwc->state & PERF_HES_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) static void bts_event_destroy(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) x86_release_hardware();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) x86_del_exclusive(x86_lbr_exclusive_bts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static int bts_event_init(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (event->attr.type != bts_pmu.type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * BTS leaks kernel addresses even when CPL0 tracing is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * disabled, so disallow intel_bts driver for unprivileged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * users on paranoid systems since it provides trace data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * to the user in a zero-copy fashion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Note that the default paranoia setting permits unprivileged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * users to profile the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (event->attr.exclude_kernel) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) ret = perf_allow_kernel(&event->attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (x86_add_exclusive(x86_lbr_exclusive_bts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) ret = x86_reserve_hardware();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) x86_del_exclusive(x86_lbr_exclusive_bts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) event->destroy = bts_event_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) static void bts_event_read(struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) static __init int bts_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (boot_cpu_has(X86_FEATURE_PTI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * BTS hardware writes through a virtual memory map we must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) * either use the kernel physical map, or the user mapping of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * the AUX buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * However, since this driver supports per-CPU and per-task inherit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * we cannot use the user mapping since it will not be available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * if we're not running the owning process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * With PTI we can't use the kernal map either, because its not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * there when we run userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * For now, disable this driver when using PTI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) bts_pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) PERF_PMU_CAP_EXCLUSIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) bts_pmu.task_ctx_nr = perf_sw_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) bts_pmu.event_init = bts_event_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) bts_pmu.add = bts_event_add;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) bts_pmu.del = bts_event_del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) bts_pmu.start = bts_event_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) bts_pmu.stop = bts_event_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) bts_pmu.read = bts_event_read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) bts_pmu.setup_aux = bts_buffer_setup_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) bts_pmu.free_aux = bts_buffer_free_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return perf_pmu_register(&bts_pmu, "intel_bts", -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) arch_initcall(bts_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags