^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Performance events ring-buffer code:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/circ_buf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/nospec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static void perf_output_wakeup(struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) atomic_set(&handle->rb->poll, EPOLLIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) handle->event->pending_wakeup = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) irq_work_queue(&handle->event->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * We need to ensure a later event_id doesn't publish a head when a former
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * event isn't done writing. However since we need to deal with NMIs we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * cannot fully serialize things.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * We only publish the head (and generate a wakeup) when the outer-most
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * event completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static void perf_output_get_handle(struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct perf_buffer *rb = handle->rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * Avoid an explicit LOAD/STORE such that architectures with memops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * can use them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) (*(volatile unsigned int *)&rb->nest)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) handle->wakeup = local_read(&rb->wakeup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static void perf_output_put_handle(struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct perf_buffer *rb = handle->rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) unsigned long head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned int nest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * If this isn't the outermost nesting, we don't have to update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * @rb->user_page->data_head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) nest = READ_ONCE(rb->nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if (nest > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) WRITE_ONCE(rb->nest, nest - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * In order to avoid publishing a head value that goes backwards,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * we must ensure the load of @rb->head happens after we've
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * incremented @rb->nest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * Otherwise we can observe a @rb->head value before one published
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * by an IRQ/NMI happening between the load and the increment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) head = local_read(&rb->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * IRQ/NMI can happen here and advance @rb->head, causing our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * load above to be stale.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Since the mmap() consumer (userspace) can run on a different CPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * kernel user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * if (LOAD ->data_tail) { LOAD ->data_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * (A) smp_rmb() (C)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * STORE $data LOAD $data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * smp_wmb() (B) smp_mb() (D)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * STORE ->data_head STORE ->data_tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * Where A pairs with D, and B pairs with C.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * In our case (A) is a control dependency that separates the load of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * the ->data_tail and the stores of $data. In case ->data_tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * indicates there is no room in the buffer to store $data we do not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * D needs to be a full barrier since it separates the data READ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * from the tail WRITE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * For B a WMB is sufficient since it separates two WRITEs, and for C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * an RMB is sufficient since it separates two READs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * See perf_output_begin().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) smp_wmb(); /* B, matches C */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) WRITE_ONCE(rb->user_page->data_head, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * We must publish the head before decrementing the nest count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * otherwise an IRQ/NMI can publish a more recent head value and our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * write will (temporarily) publish a stale value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) WRITE_ONCE(rb->nest, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Ensure we decrement @rb->nest before we validate the @rb->head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * Otherwise we cannot be sure we caught the 'last' nested update.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (unlikely(head != local_read(&rb->head))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) WRITE_ONCE(rb->nest, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (handle->wakeup != local_read(&rb->wakeup))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) perf_output_wakeup(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) ring_buffer_has_space(unsigned long head, unsigned long tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned long data_size, unsigned int size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) bool backward)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!backward)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return CIRC_SPACE(head, tail, data_size) >= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return CIRC_SPACE(tail, head, data_size) >= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) __perf_output_begin(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct perf_event *event, unsigned int size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) bool backward)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct perf_buffer *rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned long tail, offset, head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) int have_lost, page_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct perf_event_header header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) u64 id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) u64 lost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) } lost_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * For inherited events we send all the output towards the parent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (event->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) event = event->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) rb = rcu_dereference(event->rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (unlikely(!rb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (unlikely(rb->paused)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (rb->nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) local_inc(&rb->lost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) handle->rb = rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) handle->event = event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) have_lost = local_read(&rb->lost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (unlikely(have_lost)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) size += sizeof(lost_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (event->attr.sample_id_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) size += event->id_header_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) perf_output_get_handle(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) tail = READ_ONCE(rb->user_page->data_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) offset = head = local_read(&rb->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (!rb->overwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (unlikely(!ring_buffer_has_space(head, tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) perf_data_size(rb),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) size, backward)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * The above forms a control dependency barrier separating the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * @tail load above from the data stores below. Since the @tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * load is required to compute the branch to fail below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * A, matches D; the full memory barrier userspace SHOULD issue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * after reading the data and before storing the new tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * position.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * See perf_output_put_handle().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (!backward)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) head += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) head -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) } while (local_cmpxchg(&rb->head, offset, head) != offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (backward) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) offset = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) head = (u64)(-head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * We rely on the implied barrier() by local_cmpxchg() to ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * none of the data stores below can be lifted up by the compiler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) local_add(rb->watermark, &rb->wakeup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) page_shift = PAGE_SHIFT + page_order(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) handle->page = (offset >> page_shift) & (rb->nr_pages - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) offset &= (1UL << page_shift) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) handle->addr = rb->data_pages[handle->page] + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) handle->size = (1UL << page_shift) - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (unlikely(have_lost)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) lost_event.header.size = sizeof(lost_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) lost_event.header.type = PERF_RECORD_LOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) lost_event.header.misc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) lost_event.id = event->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) lost_event.lost = local_xchg(&rb->lost, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /* XXX mostly redundant; @data is already fully initializes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) perf_event_header__init_id(&lost_event.header, data, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) perf_output_put(handle, lost_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) perf_event__output_id_sample(event, handle, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) local_inc(&rb->lost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) perf_output_put_handle(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) int perf_output_begin_forward(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) struct perf_event *event, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return __perf_output_begin(handle, data, event, size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) int perf_output_begin_backward(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) struct perf_event *event, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return __perf_output_begin(handle, data, event, size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) int perf_output_begin(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct perf_sample_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct perf_event *event, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return __perf_output_begin(handle, data, event, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unlikely(is_write_backward(event)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) unsigned int perf_output_copy(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) const void *buf, unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return __output_copy(handle, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) unsigned int perf_output_skip(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return __output_skip(handle, NULL, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) void perf_output_end(struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) perf_output_put_handle(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) long max_size = perf_data_size(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (watermark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) rb->watermark = min(max_size, watermark);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!rb->watermark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) rb->watermark = max_size / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (flags & RING_BUFFER_WRITABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) rb->overwrite = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) rb->overwrite = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) refcount_set(&rb->refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) INIT_LIST_HEAD(&rb->event_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) spin_lock_init(&rb->event_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * perf_output_begin() only checks rb->paused, therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * rb->paused must be true if we have no pages for output.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (!rb->nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) rb->paused = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * OVERWRITE is determined by perf_aux_output_end() and can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * be passed in directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) handle->aux_flags |= flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) EXPORT_SYMBOL_GPL(perf_aux_output_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * This is called before hardware starts writing to the AUX area to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * obtain an output handle and make sure there's room in the buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * When the capture completes, call perf_aux_output_end() to commit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * the recorded data to the buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * The ordering is similar to that of perf_output_{begin,end}, with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * the exception of (B), which should be taken care of by the pmu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * driver, since ordering rules will differ depending on hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * Call this from pmu::start(); see the comment in perf_aux_output_end()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * about its use in pmu callbacks. Both can also be called from the PMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * handler if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) void *perf_aux_output_begin(struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) struct perf_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct perf_event *output_event = event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unsigned long aux_head, aux_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct perf_buffer *rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned int nest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (output_event->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) output_event = output_event->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * Since this will typically be open across pmu::add/pmu::del, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * grab ring_buffer's refcount instead of holding rcu read lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * to make sure it doesn't disappear under us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) rb = ring_buffer_get(output_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (!rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (!rb_has_aux(rb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * about to get freed, so we leave immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * Checking rb::aux_mmap_count and rb::refcount has to be done in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * the same order, see perf_mmap_close. Otherwise we end up freeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * aux pages in this path, which is a bug, because in_atomic().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (!atomic_read(&rb->aux_mmap_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (!refcount_inc_not_zero(&rb->aux_refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) nest = READ_ONCE(rb->aux_nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * Nesting is not supported for AUX area, make sure nested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * writers are caught early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (WARN_ON_ONCE(nest))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) goto err_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) WRITE_ONCE(rb->aux_nest, nest + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) aux_head = rb->aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) handle->rb = rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) handle->event = event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) handle->head = aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) handle->size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) handle->aux_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * In overwrite mode, AUX data stores do not depend on aux_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * therefore (A) control dependency barrier does not exist. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * (B) <-> (C) ordering is still observed by the pmu driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (!rb->aux_overwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) aux_tail = READ_ONCE(rb->user_page->aux_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (aux_head - aux_tail < perf_aux_size(rb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * handle->size computation depends on aux_tail load; this forms a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * control dependency barrier separating aux_tail load from aux data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * store that will be enabled on successful return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (!handle->size) { /* A, matches D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) event->pending_disable = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) perf_output_wakeup(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) WRITE_ONCE(rb->aux_nest, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) goto err_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return handle->rb->aux_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) err_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /* can't be last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) rb_free_aux(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) ring_buffer_put(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) handle->event = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) EXPORT_SYMBOL_GPL(perf_aux_output_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (rb->aux_overwrite)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * Commit the data written by hardware into the ring buffer by adjusting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * pmu driver's responsibility to observe ordering rules of the hardware,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * so that all the data is externally visible before this is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * Note: this has to be called from pmu::stop() callback, as the assumption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * of the AUX buffer management code is that after pmu::stop(), the AUX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * transaction must be stopped and therefore drop the AUX reference count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct perf_buffer *rb = handle->rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) unsigned long aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /* in overwrite mode, driver provides aux_head via handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (rb->aux_overwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) aux_head = handle->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) rb->aux_head = aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) aux_head = rb->aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) rb->aux_head += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * Only send RECORD_AUX if we have something useful to communicate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Note: the OVERWRITE records by themselves are not considered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * useful, as they don't communicate any *new* information,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * aside from the short-lived offset, that becomes history at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * the next event sched-in and therefore isn't useful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * The userspace that needs to copy out AUX data in overwrite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * mode should know to use user_page::aux_head for the actual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * offset. So, from now on we don't output AUX records that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * have *only* OVERWRITE flag set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) perf_event_aux_event(handle->event, aux_head, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) handle->aux_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (rb_need_aux_wakeup(rb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (wakeup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) handle->event->pending_disable = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) perf_output_wakeup(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) handle->event = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) WRITE_ONCE(rb->aux_nest, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /* can't be last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) rb_free_aux(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) ring_buffer_put(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) EXPORT_SYMBOL_GPL(perf_aux_output_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * Skip over a given number of bytes in the AUX buffer, due to, for example,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * hardware's alignment constraints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct perf_buffer *rb = handle->rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (size > handle->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) rb->aux_head += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (rb_need_aux_wakeup(rb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) perf_output_wakeup(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) handle->head = rb->aux_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) handle->size -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) EXPORT_SYMBOL_GPL(perf_aux_output_skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) void *perf_get_aux(struct perf_output_handle *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* this is only valid between perf_aux_output_begin and *_end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (!handle->event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return handle->rb->aux_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) EXPORT_SYMBOL_GPL(perf_get_aux);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * Copy out AUX data from an AUX handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) long perf_output_copy_aux(struct perf_output_handle *aux_handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) struct perf_output_handle *handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) unsigned long from, unsigned long to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) struct perf_buffer *rb = aux_handle->rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) unsigned long tocopy, remainder, len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) tocopy = PAGE_SIZE - offset_in_page(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (to > from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) tocopy = min(tocopy, to - from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (!tocopy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) addr = rb->aux_pages[from >> PAGE_SHIFT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) addr += offset_in_page(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) remainder = perf_output_copy(handle, addr, tocopy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (remainder)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) len += tocopy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) from += tocopy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) } while (to != from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static struct page *rb_alloc_aux_page(int node, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) if (order > MAX_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) order = MAX_ORDER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) page = alloc_pages_node(node, PERF_AUX_GFP, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) } while (!page && order--);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (page && order) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * Communicate the allocation size to the driver:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * if we managed to secure a high-order allocation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * set its first page's private to this order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * !PagePrivate(page) means it's just a normal page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) split_page(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) SetPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) set_page_private(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static void rb_free_aux_page(struct perf_buffer *rb, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct page *page = virt_to_page(rb->aux_pages[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) ClearPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) page->mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) static void __rb_free_aux(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) int pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * Should never happen, the last reference should be dropped from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * perf_mmap_close() path, which first stops aux transactions (which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * in turn are the atomic holders of aux_refcount) and then does the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * last rb_free_aux().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) WARN_ON_ONCE(in_atomic());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (rb->aux_priv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) rb->free_aux(rb->aux_priv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) rb->free_aux = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) rb->aux_priv = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (rb->aux_nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) for (pg = 0; pg < rb->aux_nr_pages; pg++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) rb_free_aux_page(rb, pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) kfree(rb->aux_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) rb->aux_nr_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) pgoff_t pgoff, int nr_pages, long watermark, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) bool overwrite = !(flags & RING_BUFFER_WRITABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) int ret = -ENOMEM, max_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (!has_aux(event))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * We need to start with the max_order that fits in nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * not the other way around, hence ilog2() and not get_order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) max_order = ilog2(nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * PMU requests more than one contiguous chunks of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * for SW double buffering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (!overwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if (!max_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) max_order--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (!rb->aux_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) rb->free_aux = event->pmu->free_aux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) int last, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) page = rb_alloc_aux_page(node, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) for (last = rb->aux_nr_pages + (1 << page_private(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) last > rb->aux_nr_pages; rb->aux_nr_pages++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) rb->aux_pages[rb->aux_nr_pages] = page_address(page++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * In overwrite mode, PMUs that don't support SG may not handle more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * than one contiguous allocation, since they rely on PMI to do double
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) * buffering. In this case, the entire buffer has to be one contiguous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) overwrite) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) struct page *page = virt_to_page(rb->aux_pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (page_private(page) != max_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) overwrite);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (!rb->aux_priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * aux_pages (and pmu driver's private data, aux_priv) will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * referenced in both producer's and consumer's contexts, thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * we keep a refcount here to make sure either of the two can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * reference them safely.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) refcount_set(&rb->aux_refcount, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) rb->aux_overwrite = overwrite;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) rb->aux_watermark = watermark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (!rb->aux_watermark && !rb->aux_overwrite)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) rb->aux_watermark = nr_pages << (PAGE_SHIFT - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) rb->aux_pgoff = pgoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) __rb_free_aux(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) void rb_free_aux(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (refcount_dec_and_test(&rb->aux_refcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) __rb_free_aux(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) #ifndef CONFIG_PERF_USE_VMALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * Back perf_mmap() with regular GFP_KERNEL-0 pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) static struct page *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (pgoff > rb->nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (pgoff == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) return virt_to_page(rb->user_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return virt_to_page(rb->data_pages[pgoff - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static void *perf_mmap_alloc_page(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) node = (cpu == -1) ? cpu : cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) return page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static void perf_mmap_free_page(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) struct page *page = virt_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) page->mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) struct perf_buffer *rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) size = sizeof(struct perf_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) size += nr_pages * sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (order_base_2(size) >= PAGE_SHIFT+MAX_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) rb = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (!rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) rb->user_page = perf_mmap_alloc_page(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (!rb->user_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) goto fail_user_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) rb->data_pages[i] = perf_mmap_alloc_page(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (!rb->data_pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) goto fail_data_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) rb->nr_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) ring_buffer_init(rb, watermark, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) return rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) fail_data_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) for (i--; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) perf_mmap_free_page(rb->data_pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) perf_mmap_free_page(rb->user_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) fail_user_page:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) kfree(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) void rb_free(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) perf_mmap_free_page(rb->user_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) for (i = 0; i < rb->nr_pages; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) perf_mmap_free_page(rb->data_pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) kfree(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) static int data_page_nr(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) return rb->nr_pages << page_order(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) static struct page *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) __perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /* The '>' counts in the user page. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (pgoff > data_page_nr(rb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static void perf_mmap_unmark_page(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct page *page = vmalloc_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) page->mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) static void rb_free_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) struct perf_buffer *rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) void *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) int i, nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) rb = container_of(work, struct perf_buffer, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) nr = data_page_nr(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) base = rb->user_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /* The '<=' counts in the user page. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) for (i = 0; i <= nr; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) perf_mmap_unmark_page(base + (i * PAGE_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) vfree(base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) kfree(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) void rb_free(struct perf_buffer *rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) schedule_work(&rb->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct perf_buffer *rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) void *all_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) size = sizeof(struct perf_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) size += sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) rb = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) if (!rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) INIT_WORK(&rb->work, rb_free_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (!all_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) goto fail_all_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) rb->user_page = all_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) rb->data_pages[0] = all_buf + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) if (nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) rb->nr_pages = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) rb->page_order = ilog2(nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) ring_buffer_init(rb, watermark, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) return rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) fail_all_buf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) kfree(rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) struct page *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (rb->aux_nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) /* above AUX space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) if (pgoff > rb->aux_pgoff + rb->aux_nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /* AUX space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (pgoff >= rb->aux_pgoff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return virt_to_page(rb->aux_pages[aux_pgoff]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) return __perf_mmap_to_page(rb, pgoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags