^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * arch/arm64/kernel/entry-ftrace.S
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013 Linaro Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * the regular function prologue. For an enabled callsite, ftrace_init_nop() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * ftrace_make_call() have patched those NOPs to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * MOV X9, LR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * BL <entry>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * clobber.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * We save the callsite's context into a pt_regs before invoking any ftrace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * callbacks. So that we can get a sensible backtrace, we create a stack record
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * for the callsite and the ftrace entry assembly. This is not sufficient for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * reliable stacktrace: until we create the callsite stack record, its caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * is missing from the LR and existing chain of frame records.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) .macro ftrace_regs_entry, allregs=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /* Make room for pt_regs, plus a callee frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) sub sp, sp, #(S_FRAME_SIZE + 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* Save function arguments (and x9 for simplicity) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) stp x0, x1, [sp, #S_X0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) stp x2, x3, [sp, #S_X2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) stp x4, x5, [sp, #S_X4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) stp x6, x7, [sp, #S_X6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) stp x8, x9, [sp, #S_X8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* Optionally save the callee-saved registers, always save the FP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) .if \allregs == 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) stp x10, x11, [sp, #S_X10]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) stp x12, x13, [sp, #S_X12]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) stp x14, x15, [sp, #S_X14]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) stp x16, x17, [sp, #S_X16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) stp x18, x19, [sp, #S_X18]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) stp x20, x21, [sp, #S_X20]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) stp x22, x23, [sp, #S_X22]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) stp x24, x25, [sp, #S_X24]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) stp x26, x27, [sp, #S_X26]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) stp x28, x29, [sp, #S_X28]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) .else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) str x29, [sp, #S_FP]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) .endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* Save the callsite's SP and LR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) add x10, sp, #(S_FRAME_SIZE + 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) stp x9, x10, [sp, #S_LR]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* Save the PC after the ftrace callsite */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) str x30, [sp, #S_PC]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* Create a frame record for the callsite above pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) stp x29, x9, [sp, #S_FRAME_SIZE]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) add x29, sp, #S_FRAME_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /* Create our frame record within pt_regs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) stp x29, x30, [sp, #S_STACKFRAME]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) add x29, sp, #S_STACKFRAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) SYM_CODE_START(ftrace_regs_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #ifdef BTI_C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) BTI_C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) ftrace_regs_entry 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) b ftrace_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) SYM_CODE_END(ftrace_regs_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) SYM_CODE_START(ftrace_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #ifdef BTI_C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) BTI_C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ftrace_regs_entry 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) b ftrace_common
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) SYM_CODE_END(ftrace_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) SYM_CODE_START(ftrace_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) mov x1, x9 // parent_ip (callsite's LR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) ldr_l x2, function_trace_op // op
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) mov x3, sp // regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) bl ftrace_stub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) nop // If enabled, this will be replaced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) // "b ftrace_graph_caller"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * to restore x0-x8, x29, and x30.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) ftrace_common_return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* Restore function arguments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ldp x0, x1, [sp]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ldp x2, x3, [sp, #S_X2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) ldp x4, x5, [sp, #S_X4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) ldp x6, x7, [sp, #S_X6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) ldr x8, [sp, #S_X8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* Restore the callsite's FP, LR, PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) ldr x29, [sp, #S_FP]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) ldr x30, [sp, #S_LR]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) ldr x9, [sp, #S_PC]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* Restore the callsite's SP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) add sp, sp, #S_FRAME_SIZE + 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ret x9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) SYM_CODE_END(ftrace_common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) SYM_CODE_START(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) ldr x0, [sp, #S_PC]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) add x1, sp, #S_LR // parent_ip (callsite's LR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) ldr x2, [sp, #S_FRAME_SIZE] // parent fp (callsite's FP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) bl prepare_ftrace_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) b ftrace_common_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) SYM_CODE_END(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * Gcc with -pg will put the following code in the beginning of each function:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * mov x0, x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * bl _mcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * [function's body ...]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * ftrace is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * Please note that x0 as an argument will not be used here because we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * get lr(x30) of instrumented function at any time by winding up call stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * as long as the kernel is compiled without -fomit-frame-pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * (or CONFIG_FRAME_POINTER, this is forced on arm64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * stack layout after mcount_enter in _mcount():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * current sp/fp => 0:+-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * in _mcount() | x29 | -> instrumented function's fp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * +-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * | x30 | -> _mcount()'s lr (= instrumented function's pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * old sp => +16:+-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * when instrumented | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * function calls | ... |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * _mcount() | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * instrumented => +xx:+-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * function's fp | x29 | -> parent's fp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * +-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * | x30 | -> instrumented function's lr (= parent's pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * +-----+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * | ... |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) .macro mcount_enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) .macro mcount_exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .macro mcount_adjust_addr rd, rn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) sub \rd, \rn, #AARCH64_INSN_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* for instrumented function's parent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) .macro mcount_get_parent_fp reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) ldr \reg, [x29]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) ldr \reg, [\reg]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* for instrumented function */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) .macro mcount_get_pc0 reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) mcount_adjust_addr \reg, x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) .macro mcount_get_pc reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) ldr \reg, [x29, #8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) mcount_adjust_addr \reg, \reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) .macro mcount_get_lr reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) ldr \reg, [x29]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) ldr \reg, [\reg, #8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) .macro mcount_get_lr_addr reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ldr \reg, [x29]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) add \reg, \reg, #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #ifndef CONFIG_DYNAMIC_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * void _mcount(unsigned long return_address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * @return_address: return address to instrumented function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * This function makes calls, if enabled, to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * - tracer function to probe instrumented function's entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * - ftrace_graph_caller to set up an exit hook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) SYM_FUNC_START(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) mcount_enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) ldr_l x2, ftrace_trace_function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) adr x0, ftrace_stub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) cmp x0, x2 // if (ftrace_trace_function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) b.eq skip_ftrace_call // != ftrace_stub) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) mcount_get_pc x0 // function's pc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) mcount_get_lr x1 // function's lr (= parent's pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) blr x2 // (*ftrace_trace_function)(pc, lr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) skip_ftrace_call: // }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ldr_l x2, ftrace_graph_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) cmp x0, x2 // if ((ftrace_graph_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) b.ne ftrace_graph_caller // != ftrace_stub)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) cmp x0, x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) b.ne ftrace_graph_caller // ftrace_graph_caller();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) mcount_exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) SYM_FUNC_END(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) EXPORT_SYMBOL(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) NOKPROBE(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #else /* CONFIG_DYNAMIC_FTRACE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * _mcount() is used to build the kernel with -pg option, but all the branch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * instructions to _mcount() are replaced to NOP initially at kernel start up,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * and later on, NOP to branch to ftrace_caller() when enabled or branch to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * NOP when disabled per-function base.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) SYM_FUNC_START(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) SYM_FUNC_END(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) EXPORT_SYMBOL(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) NOKPROBE(_mcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * void ftrace_caller(unsigned long return_address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * @return_address: return address to instrumented function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * This function is a counterpart of _mcount() in 'static' ftrace, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * makes calls to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * - tracer function to probe instrumented function's entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * - ftrace_graph_caller to set up an exit hook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) SYM_FUNC_START(ftrace_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) mcount_enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) mcount_get_pc0 x0 // function's pc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) mcount_get_lr x1 // function's lr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) nop // This will be replaced with "bl xxx"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) // where xxx can be any kind of tracer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) nop // If enabled, this will be replaced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) // "b ftrace_graph_caller"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) mcount_exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) SYM_FUNC_END(ftrace_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #endif /* CONFIG_DYNAMIC_FTRACE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * void ftrace_graph_caller(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * Called from _mcount() or ftrace_caller() when function_graph tracer is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * selected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * This function w/ prepare_ftrace_return() fakes link register's value on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * the call stack in order to intercept instrumented function's return path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * and run return_to_handler() later on its exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) SYM_FUNC_START(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) mcount_get_pc x0 // function's pc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) mcount_get_lr_addr x1 // pointer to function's saved lr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) mcount_get_parent_fp x2 // parent's fp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) mcount_exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) SYM_FUNC_END(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) SYM_FUNC_START(ftrace_stub)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) SYM_FUNC_END(ftrace_stub)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * void return_to_handler(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * Run ftrace_return_to_handler() before going back to parent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * @fp is checked against the value passed by ftrace_graph_caller().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) SYM_CODE_START(return_to_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /* save return value regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) sub sp, sp, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) stp x0, x1, [sp]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) stp x2, x3, [sp, #16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) stp x4, x5, [sp, #32]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) stp x6, x7, [sp, #48]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) mov x0, x29 // parent's fp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) mov x30, x0 // restore the original return address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) /* restore return value regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) ldp x0, x1, [sp]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) ldp x2, x3, [sp, #16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) ldp x4, x5, [sp, #32]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) ldp x6, x7, [sp, #48]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) add sp, sp, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) SYM_CODE_END(return_to_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags