^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * thread-stack.c: Synthesize a thread's stack using call / return events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2014, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/rbtree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/zalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <stdlib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "thread.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "event.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "machine.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "env.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "debug.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "symbol.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "comm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "call-path.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "thread-stack.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define STACK_GROWTH 2048
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * State of retpoline detection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * RETPOLINE_NONE: no retpoline detection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * X86_RETPOLINE_POSSIBLE: x86 retpoline possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * X86_RETPOLINE_DETECTED: x86 retpoline detected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) enum retpoline_state_t {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) RETPOLINE_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) X86_RETPOLINE_POSSIBLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) X86_RETPOLINE_DETECTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * struct thread_stack_entry - thread stack entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * @ret_addr: return address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * @timestamp: timestamp (if known)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * @ref: external reference (e.g. db_id of sample)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * @branch_count: the branch count when the entry was created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * @insn_count: the instruction count when the entry was created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * @cyc_count the cycle count when the entry was created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * @db_id: id used for db-export
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * @cp: call path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * @no_call: a 'call' was not seen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * @trace_end: a 'call' but trace ended
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * @non_call: a branch but not a 'call' to the start of a different symbol
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct thread_stack_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u64 ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) u64 timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) u64 ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) u64 branch_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) u64 insn_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) u64 cyc_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) u64 db_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) bool no_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) bool trace_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) bool non_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * struct thread_stack - thread stack constructed from 'call' and 'return'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * branch samples.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * @stack: array that holds the stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * @cnt: number of entries in the stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * @sz: current maximum stack size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @trace_nr: current trace number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * @branch_count: running branch count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * @insn_count: running instruction count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * @cyc_count running cycle count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * @kernel_start: kernel start address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * @last_time: last timestamp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * @crp: call/return processor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * @comm: current comm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * @arr_sz: size of array if this is the first element of an array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * @rstate: used to detect retpolines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * @br_stack_rb: branch stack (ring buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * @br_stack_sz: maximum branch stack size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * @br_stack_pos: current position in @br_stack_rb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * @mispred_all: mark all branches as mispredicted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct thread_stack {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct thread_stack_entry *stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) size_t cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) size_t sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) u64 trace_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) u64 branch_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) u64 insn_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) u64 cyc_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) u64 kernel_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) u64 last_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct call_return_processor *crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct comm *comm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned int arr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) enum retpoline_state_t rstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct branch_stack *br_stack_rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned int br_stack_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned int br_stack_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) bool mispred_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Assume pid == tid == 0 identifies the idle task as defined by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * perf_session__register_idle_thread(). The idle task is really 1 task per cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * and therefore requires a stack for each cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static inline bool thread_stack__per_cpu(struct thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return !(thread->tid || thread->pid_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static int thread_stack__grow(struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct thread_stack_entry *new_stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) size_t sz, new_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) new_sz = ts->sz + STACK_GROWTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) sz = new_sz * sizeof(struct thread_stack_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) new_stack = realloc(ts->stack, sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (!new_stack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ts->stack = new_stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ts->sz = new_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static int thread_stack__init(struct thread_stack *ts, struct thread *thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct call_return_processor *crp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) bool callstack, unsigned int br_stack_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (callstack) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) err = thread_stack__grow(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (br_stack_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) size_t sz = sizeof(struct branch_stack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) sz += br_stack_sz * sizeof(struct branch_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) ts->br_stack_rb = zalloc(sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (!ts->br_stack_rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ts->br_stack_sz = br_stack_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) if (thread->maps && thread->maps->machine) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct machine *machine = thread->maps->machine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) const char *arch = perf_env__arch(machine->env);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) ts->kernel_start = machine__kernel_start(machine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (!strcmp(arch, "x86"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) ts->rstate = X86_RETPOLINE_POSSIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) ts->kernel_start = 1ULL << 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) ts->crp = crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static struct thread_stack *thread_stack__new(struct thread *thread, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct call_return_processor *crp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) bool callstack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) unsigned int br_stack_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct thread_stack *ts = thread->ts, *new_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned int old_sz = ts ? ts->arr_sz : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) unsigned int new_sz = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (thread_stack__per_cpu(thread) && cpu > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) new_sz = roundup_pow_of_two(cpu + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (!ts || new_sz > old_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) new_ts = calloc(new_sz, sizeof(*ts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (!new_ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) memcpy(new_ts, ts, old_sz * sizeof(*ts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) new_ts->arr_sz = new_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) zfree(&thread->ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) thread->ts = new_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) ts = new_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (thread_stack__per_cpu(thread) && cpu > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) (unsigned int)cpu < ts->arr_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) ts += cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (!ts->stack &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) thread_stack__init(ts, thread, crp, callstack, br_stack_sz))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static struct thread_stack *thread__cpu_stack(struct thread *thread, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct thread_stack *ts = thread->ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (cpu < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) cpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (!ts || (unsigned int)cpu >= ts->arr_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) ts += cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (!ts->stack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static inline struct thread_stack *thread__stack(struct thread *thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (thread_stack__per_cpu(thread))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return thread__cpu_stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return thread->ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) bool trace_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (ts->cnt == ts->sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) err = thread_stack__grow(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) pr_warning("Out of memory: discarding thread stack\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) ts->cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) ts->stack[ts->cnt].trace_end = trace_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) ts->stack[ts->cnt++].ret_addr = ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * In some cases there may be functions which are not seen to return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * For example when setjmp / longjmp has been used. Or the perf context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * switch in the kernel which doesn't stop and start tracing in exactly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * the same code path. When that happens the return address will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * further down the stack. If the return address is not found at all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * we assume the opposite (i.e. this is a return for a call that wasn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * seen for some reason) and leave the stack alone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) for (i = ts->cnt; i; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (ts->stack[--i].ret_addr == ret_addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) ts->cnt = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static void thread_stack__pop_trace_end(struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) for (i = ts->cnt; i; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (ts->stack[--i].trace_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ts->cnt = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static bool thread_stack__in_kernel(struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (!ts->cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return ts->stack[ts->cnt - 1].cp->in_kernel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static int thread_stack__call_return(struct thread *thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct thread_stack *ts, size_t idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) u64 timestamp, u64 ref, bool no_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct call_return_processor *crp = ts->crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct thread_stack_entry *tse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) struct call_return cr = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) .thread = thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) .comm = ts->comm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) .db_id = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) u64 *parent_db_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) tse = &ts->stack[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) cr.cp = tse->cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) cr.call_time = tse->timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) cr.return_time = timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) cr.branch_count = ts->branch_count - tse->branch_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) cr.insn_count = ts->insn_count - tse->insn_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) cr.cyc_count = ts->cyc_count - tse->cyc_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) cr.db_id = tse->db_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) cr.call_ref = tse->ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) cr.return_ref = ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (tse->no_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) cr.flags |= CALL_RETURN_NO_CALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (no_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) cr.flags |= CALL_RETURN_NO_RETURN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (tse->non_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) cr.flags |= CALL_RETURN_NON_CALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * The parent db_id must be assigned before exporting the child. Note
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * it is not possible to export the parent first because its information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * is not yet complete because its 'return' has not yet been processed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) parent_db_id = idx ? &(tse - 1)->db_id : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return crp->process(&cr, parent_db_id, crp->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct call_return_processor *crp = ts->crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!crp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) ts->cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) ts->br_stack_pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (ts->br_stack_rb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ts->br_stack_rb->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) while (ts->cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) err = thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) ts->last_time, 0, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) pr_err("Error flushing thread stack!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) ts->cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) int thread_stack__flush(struct thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct thread_stack *ts = thread->ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned int pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) for (pos = 0; pos < ts->arr_sz; pos++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) int ret = __thread_stack__flush(thread, ts + pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) err = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) u64 from_ip, u64 to_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) struct branch_stack *bs = ts->br_stack_rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct branch_entry *be;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (!ts->br_stack_pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) ts->br_stack_pos = ts->br_stack_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) ts->br_stack_pos -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) be = &bs->entries[ts->br_stack_pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) be->from = from_ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) be->to = to_ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) be->flags.value = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) be->flags.abort = !!(flags & PERF_IP_FLAG_TX_ABORT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) be->flags.in_tx = !!(flags & PERF_IP_FLAG_IN_TX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /* No support for mispredict */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) be->flags.mispred = ts->mispred_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (bs->nr < ts->br_stack_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) bs->nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) int thread_stack__event(struct thread *thread, int cpu, u32 flags, u64 from_ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) u64 to_ip, u16 insn_len, u64 trace_nr, bool callstack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) unsigned int br_stack_sz, bool mispred_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (!thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (!ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) pr_warning("Out of memory: no thread stack\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) ts->trace_nr = trace_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) ts->mispred_all = mispred_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * When the trace is discontinuous, the trace_nr changes. In that case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * the stack might be completely invalid. Better to report nothing than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * to report something misleading, so flush the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (trace_nr != ts->trace_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (ts->trace_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __thread_stack__flush(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) ts->trace_nr = trace_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (br_stack_sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) thread_stack__update_br_stack(ts, flags, from_ip, to_ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * Stop here if thread_stack__process() is in use, or not recording call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (ts->crp || !callstack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (flags & PERF_IP_FLAG_CALL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) u64 ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (!to_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) ret_addr = from_ip + insn_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (ret_addr == to_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) return 0; /* Zero-length calls are excluded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return thread_stack__push(ts, ret_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) flags & PERF_IP_FLAG_TRACE_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) } else if (flags & PERF_IP_FLAG_TRACE_BEGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * If the caller did not change the trace number (which would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * have flushed the stack) then try to make sense of the stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * Possibly, tracing began after returning to the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * address, so try to pop that. Also, do not expect a call made
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * when the trace ended, to return, so pop that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) thread_stack__pop(ts, to_ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) thread_stack__pop_trace_end(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) } else if ((flags & PERF_IP_FLAG_RETURN) && from_ip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) thread_stack__pop(ts, to_ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) void thread_stack__set_trace_nr(struct thread *thread, int cpu, u64 trace_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (trace_nr != ts->trace_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (ts->trace_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) __thread_stack__flush(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) ts->trace_nr = trace_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) __thread_stack__flush(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) zfree(&ts->stack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) zfree(&ts->br_stack_rb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) unsigned int arr_sz = ts->arr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) __thread_stack__free(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) memset(ts, 0, sizeof(*ts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) ts->arr_sz = arr_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) void thread_stack__free(struct thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct thread_stack *ts = thread->ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) unsigned int pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) for (pos = 0; pos < ts->arr_sz; pos++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) __thread_stack__free(thread, ts + pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) zfree(&thread->ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static inline u64 callchain_context(u64 ip, u64 kernel_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ip < kernel_start ? PERF_CONTEXT_USER : PERF_CONTEXT_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) void thread_stack__sample(struct thread *thread, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct ip_callchain *chain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) size_t sz, u64 ip, u64 kernel_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) u64 context = callchain_context(ip, kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) u64 last_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) size_t i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (sz < 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) chain->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) chain->ips[0] = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) chain->ips[1] = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (!ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) chain->nr = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) last_context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) ip = ts->stack[ts->cnt - j].ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) context = callchain_context(ip, kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (context != last_context) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (i >= sz - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) chain->ips[i++] = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) last_context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) chain->ips[i] = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) chain->nr = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * Hardware sample records, created some time after the event occurred, need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * have subsequent addresses removed from the call chain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) void thread_stack__sample_late(struct thread *thread, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct ip_callchain *chain, size_t sz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) u64 sample_ip, u64 kernel_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) u64 sample_context = callchain_context(sample_ip, kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) u64 last_context, context, ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) size_t nr = 0, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) if (sz < 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) chain->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * When tracing kernel space, kernel addresses occur at the top of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * call chain after the event occurred but before tracing stopped.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * Skip them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) for (j = 1; j <= ts->cnt; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) ip = ts->stack[ts->cnt - j].ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) context = callchain_context(ip, kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (context == PERF_CONTEXT_USER ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) (context == sample_context && ip == sample_ip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) last_context = sample_ip; /* Use sample_ip as an invalid context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) for (; nr < sz && j <= ts->cnt; nr++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ip = ts->stack[ts->cnt - j].ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) context = callchain_context(ip, kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (context != last_context) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (nr >= sz - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) chain->ips[nr++] = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) last_context = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) chain->ips[nr] = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) chain->nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) chain->ips[0] = sample_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) chain->ips[1] = sample_ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) chain->nr = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) void thread_stack__br_sample(struct thread *thread, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct branch_stack *dst, unsigned int sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) const size_t bsz = sizeof(struct branch_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct branch_stack *src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct branch_entry *be;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) unsigned int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) dst->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) src = ts->br_stack_rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (!src->nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) dst->nr = min((unsigned int)src->nr, sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) be = &dst->entries[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (src->nr >= ts->br_stack_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) sz -= nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) be = &dst->entries[nr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) nr = min(ts->br_stack_pos, sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) /* Start of user space branch entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) static bool us_start(struct branch_entry *be, u64 kernel_start, bool *start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (!*start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) *start = be->to && be->to < kernel_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return *start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * Start of branch entries after the ip fell in between 2 branches, or user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * space branch entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) static bool ks_start(struct branch_entry *be, u64 sample_ip, u64 kernel_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) bool *start, struct branch_entry *nb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (!*start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) *start = (nb && sample_ip >= be->to && sample_ip <= nb->from) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) be->from < kernel_start ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) (be->to && be->to < kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) return *start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * Hardware sample records, created some time after the event occurred, need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * have subsequent addresses removed from the branch stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) void thread_stack__br_sample_late(struct thread *thread, int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) struct branch_stack *dst, unsigned int sz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) u64 ip, u64 kernel_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) struct branch_entry *d, *s, *spos, *ssz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct branch_stack *src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) unsigned int nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) bool start = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) dst->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) src = ts->br_stack_rb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (!src->nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) spos = &src->entries[ts->br_stack_pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) ssz = &src->entries[ts->br_stack_sz];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) d = &dst->entries[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) s = spos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (ip < kernel_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * User space sample: start copying branch entries when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * branch is in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) for (s = spos; s < ssz && nr < sz; s++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) if (us_start(s, kernel_start, &start)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) *d++ = *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (src->nr >= ts->br_stack_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) for (s = &src->entries[0]; s < spos && nr < sz; s++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (us_start(s, kernel_start, &start)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) *d++ = *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) struct branch_entry *nb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * Kernel space sample: start copying branch entries when the ip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * falls in between 2 branches (or the branch is in user space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * because then the start must have been missed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) for (s = spos; s < ssz && nr < sz; s++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (ks_start(s, ip, kernel_start, &start, nb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) *d++ = *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) nb = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (src->nr >= ts->br_stack_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) for (s = &src->entries[0]; s < spos && nr < sz; s++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (ks_start(s, ip, kernel_start, &start, nb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) *d++ = *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) nb = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) dst->nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) struct call_return_processor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) call_return_processor__new(int (*process)(struct call_return *cr, u64 *parent_db_id, void *data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) struct call_return_processor *crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) crp = zalloc(sizeof(struct call_return_processor));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (!crp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) crp->cpr = call_path_root__new();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (!crp->cpr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) crp->process = process;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) crp->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) return crp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) free(crp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) void call_return_processor__free(struct call_return_processor *crp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (crp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) call_path_root__free(crp->cpr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) free(crp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) u64 timestamp, u64 ref, struct call_path *cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) bool no_call, bool trace_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct thread_stack_entry *tse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (!cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (ts->cnt == ts->sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) err = thread_stack__grow(ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) tse = &ts->stack[ts->cnt++];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) tse->ret_addr = ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) tse->timestamp = timestamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) tse->ref = ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) tse->branch_count = ts->branch_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) tse->insn_count = ts->insn_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) tse->cyc_count = ts->cyc_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) tse->cp = cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) tse->no_call = no_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) tse->trace_end = trace_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) tse->non_call = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) tse->db_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) u64 ret_addr, u64 timestamp, u64 ref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct symbol *sym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (!ts->cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (ts->cnt == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct thread_stack_entry *tse = &ts->stack[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (tse->cp->sym == sym)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) return thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) timestamp, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) !ts->stack[ts->cnt - 1].non_call) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) return thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) timestamp, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) size_t i = ts->cnt - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) while (i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (ts->stack[i].ret_addr != ret_addr ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) ts->stack[i].non_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) i += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) while (ts->cnt > i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) err = thread_stack__call_return(thread, ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) timestamp, ref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) timestamp, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) static int thread_stack__bottom(struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) struct perf_sample *sample,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) struct addr_location *from_al,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) struct addr_location *to_al, u64 ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) struct symbol *sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) u64 ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (sample->ip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) ip = sample->ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) sym = from_al->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) } else if (sample->addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) ip = sample->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) sym = to_al->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) cp = call_path__findnew(cpr, &cpr->call_path, sym, ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) ts->kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) return thread_stack__push_cp(ts, ip, sample->time, ref, cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) struct perf_sample *sample, u64 ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) u64 tm = sample->time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /* Return to userspace, so pop all kernel addresses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) while (thread_stack__in_kernel(ts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) err = thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) tm, ref, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) static int thread_stack__no_call_return(struct thread *thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) struct perf_sample *sample,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) struct addr_location *from_al,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct addr_location *to_al, u64 ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) struct call_path *root = &cpr->call_path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) struct symbol *fsym = from_al->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) struct symbol *tsym = to_al->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) struct call_path *cp, *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) u64 ks = ts->kernel_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) u64 addr = sample->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) u64 tm = sample->time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) u64 ip = sample->ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) if (ip >= ks && addr < ks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /* Return to userspace, so pop all kernel addresses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) err = thread_stack__pop_ks(thread, ts, sample, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /* If the stack is empty, push the userspace address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) if (!ts->cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) cp = call_path__findnew(cpr, root, tsym, addr, ks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) return thread_stack__push_cp(ts, 0, tm, ref, cp, true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) } else if (thread_stack__in_kernel(ts) && ip < ks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) /* Return to userspace, so pop all kernel addresses */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) err = thread_stack__pop_ks(thread, ts, sample, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (ts->cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) parent = ts->stack[ts->cnt - 1].cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) parent = root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (parent->sym == from_al->sym) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * At the bottom of the stack, assume the missing 'call' was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) * before the trace started. So, pop the current symbol and push
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * the 'to' symbol.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) if (ts->cnt == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) err = thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) tm, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (!ts->cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) cp = call_path__findnew(cpr, root, tsym, addr, ks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return thread_stack__push_cp(ts, addr, tm, ref, cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * Otherwise assume the 'return' is being used as a jump (e.g.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) * retpoline) and just push the 'to' symbol.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) cp = call_path__findnew(cpr, parent, tsym, addr, ks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) ts->stack[ts->cnt - 1].non_call = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * Assume 'parent' has not yet returned, so push 'to', and then push and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * pop 'from'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) cp = call_path__findnew(cpr, parent, tsym, addr, ks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) cp = call_path__findnew(cpr, cp, fsym, ip, ks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) static int thread_stack__trace_begin(struct thread *thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) struct thread_stack *ts, u64 timestamp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) u64 ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) struct thread_stack_entry *tse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) if (!ts->cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) /* Pop trace end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) tse = &ts->stack[ts->cnt - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) if (tse->trace_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) err = thread_stack__call_return(thread, ts, --ts->cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) timestamp, ref, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) static int thread_stack__trace_end(struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct perf_sample *sample, u64 ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) u64 ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) /* No point having 'trace end' on the bottom of the stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) ts->kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) ret_addr = sample->ip + sample->insn_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) false, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) static bool is_x86_retpoline(const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) const char *p = strstr(name, "__x86_indirect_thunk_");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) return p == name || !strcmp(name, "__indirect_thunk_start");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * x86 retpoline functions pollute the call graph. This function removes them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) * This does not handle function return thunks, nor is there any improvement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * for the handling of inline thunks or extern thunks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) static int thread_stack__x86_retpoline(struct thread_stack *ts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) struct perf_sample *sample,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) struct addr_location *to_al)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) struct symbol *sym = tse->cp->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) struct symbol *tsym = to_al->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (sym && is_x86_retpoline(sym->name)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) * This is a x86 retpoline fn. It pollutes the call graph by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * showing up everywhere there is an indirect branch, but does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * not itself mean anything. Here the top-of-stack is removed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * by decrementing the stack count, and then further down, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) * resulting top-of-stack is replaced with the actual target.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * The result is that the retpoline functions will no longer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) * appear in the call graph. Note this only affects the call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) * graph, since all the original branches are left unchanged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) ts->cnt -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) sym = ts->stack[ts->cnt - 2].cp->sym;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) if (sym && sym == tsym && to_al->addr != tsym->start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) * Target is back to the middle of the symbol we came
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * from so assume it is an indirect jmp and forget it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * altogether.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) ts->cnt -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) } else if (sym && sym == tsym) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * Target is back to the symbol we came from so assume it is an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * indirect jmp and forget it altogether.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) ts->cnt -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) sample->addr, ts->kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) if (!cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) /* Replace the top-of-stack with the actual target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) ts->stack[ts->cnt - 1].cp = cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) int thread_stack__process(struct thread *thread, struct comm *comm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) struct perf_sample *sample,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) struct addr_location *from_al,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) struct addr_location *to_al, u64 ref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) struct call_return_processor *crp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) struct thread_stack *ts = thread__stack(thread, sample->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) enum retpoline_state_t rstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (ts && !ts->crp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) /* Supersede thread_stack__event() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) thread_stack__reset(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) ts = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) if (!ts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) ts = thread_stack__new(thread, sample->cpu, crp, true, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) ts->comm = comm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) rstate = ts->rstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) if (rstate == X86_RETPOLINE_DETECTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ts->rstate = X86_RETPOLINE_POSSIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) /* Flush stack on exec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (ts->comm != comm && thread->pid_ == thread->tid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) err = __thread_stack__flush(thread, ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) ts->comm = comm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) /* If the stack is empty, put the current symbol on the stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) if (!ts->cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) err = thread_stack__bottom(ts, sample, from_al, to_al, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) ts->branch_count += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) ts->insn_count += sample->insn_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) ts->cyc_count += sample->cyc_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) ts->last_time = sample->time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (sample->flags & PERF_IP_FLAG_CALL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) bool trace_end = sample->flags & PERF_IP_FLAG_TRACE_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) u64 ret_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (!sample->ip || !sample->addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) ret_addr = sample->ip + sample->insn_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (ret_addr == sample->addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return 0; /* Zero-length calls are excluded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) to_al->sym, sample->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) ts->kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) cp, false, trace_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * A call to the same symbol but not the start of the symbol,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * may be the start of a x86 retpoline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) if (!err && rstate == X86_RETPOLINE_POSSIBLE && to_al->sym &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) from_al->sym == to_al->sym &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) to_al->addr != to_al->sym->start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) ts->rstate = X86_RETPOLINE_DETECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) } else if (sample->flags & PERF_IP_FLAG_RETURN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (!sample->addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) u32 return_from_kernel = PERF_IP_FLAG_SYSCALLRET |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) PERF_IP_FLAG_INTERRUPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (!(sample->flags & return_from_kernel))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) /* Pop kernel stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) return thread_stack__pop_ks(thread, ts, sample, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (!sample->ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) /* x86 retpoline 'return' doesn't match the stack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) ts->stack[ts->cnt - 1].ret_addr != sample->addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) return thread_stack__x86_retpoline(ts, sample, to_al);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) err = thread_stack__pop_cp(thread, ts, sample->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) sample->time, ref, from_al->sym);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) err = thread_stack__no_call_return(thread, ts, sample,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) from_al, to_al, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) } else if (sample->flags & PERF_IP_FLAG_TRACE_BEGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) err = thread_stack__trace_begin(thread, ts, sample->time, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) } else if (sample->flags & PERF_IP_FLAG_TRACE_END) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) err = thread_stack__trace_end(ts, sample, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) } else if (sample->flags & PERF_IP_FLAG_BRANCH &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) from_al->sym != to_al->sym && to_al->sym &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) to_al->addr == to_al->sym->start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) struct call_path_root *cpr = ts->crp->cpr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) struct call_path *cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * The compiler might optimize a call/ret combination by making
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * it a jmp. Make that visible by recording on the stack a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * branch to the start of a different symbol. Note, that means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * when a ret pops the stack, all jmps must be popped off first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) to_al->sym, sample->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) ts->kernel_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) ts->stack[ts->cnt - 1].non_call = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) size_t thread_stack__depth(struct thread *thread, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) struct thread_stack *ts = thread__stack(thread, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (!ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) return ts->cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags