Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *  Copyright (C) 2014  Steven Rostedt, Red Hat Inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/nospec-branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/unwind_hints.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/frame.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 	.code64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 	.section .text, "ax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #ifdef CONFIG_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) /* Save parent and function stack frames (rip and rbp) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #  define MCOUNT_FRAME_SIZE	(8+16*2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) /* No need to save a stack frame */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) # define MCOUNT_FRAME_SIZE	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #endif /* CONFIG_FRAME_POINTER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) /* Size of stack used to save mcount regs in save_mcount_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #define MCOUNT_REG_SIZE		(FRAME_SIZE + MCOUNT_FRAME_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * gcc -pg option adds a call to 'mcount' in most functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  * When -mfentry is used, the call is to 'fentry' and not 'mcount'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  * and is done before the function's stack frame is set up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  * They both require a set of regs to be saved before calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  * any C code and restored before returning back to the function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * On boot up, all these calls are converted into nops. When tracing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  * is enabled, the call can jump to either ftrace_caller or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * ftrace_regs_caller. Callbacks (tracing functions) that require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * it. For this reason, the size of the pt_regs structure will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * allocated on the stack and the required mcount registers will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  * be saved in the locations that pt_regs has them in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * @added: the amount of stack added before calling this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  * After this is called, the following registers contain:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  *  %rdi - holds the address that called the trampoline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  *  %rsi - holds the parent function (traced function's return address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  *  %rdx - holds the original %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) .macro save_mcount_regs added=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #ifdef CONFIG_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	/* Save the original rbp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	pushq %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	 * Stack traces will stop at the ftrace trampoline if the frame pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	 * is not set up properly. If fentry is used, we need to save a frame
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	 * pointer for the parent as well as the function traced, because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	 * fentry is called before the stack frame is set up, where as mcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	 * is called afterward.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	/* Save the parent pointer (skip orig rbp and our return address) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	pushq \added+8*2(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	pushq %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	movq %rsp, %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	/* Save the return address (now skip orig rbp, rbp and parent) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	pushq \added+8*3(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	pushq %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	movq %rsp, %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #endif /* CONFIG_FRAME_POINTER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	 * We add enough stack to save all regs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	subq $(FRAME_SIZE), %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	movq %rax, RAX(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	movq %rcx, RCX(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	movq %rdx, RDX(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	movq %rsi, RSI(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	movq %rdi, RDI(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	movq %r8, R8(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	movq %r9, R9(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	movq $0, ORIG_RAX(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	 * Save the original RBP. Even though the mcount ABI does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	 * require this, it helps out callers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) #ifdef CONFIG_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	movq MCOUNT_REG_SIZE-8(%rsp), %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	movq %rbp, %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	movq %rdx, RBP(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	/* Copy the parent address into %rsi (second parameter) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	 /* Move RIP to its proper location */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	movq %rdi, RIP(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	 * Now %rdi (the first parameter) has the return address of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	 * where ftrace_call returns. But the callbacks expect the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	 * address of the call itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	subq $MCOUNT_INSN_SIZE, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) .macro restore_mcount_regs save=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	/* ftrace_regs_caller or frame pointers require this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	movq RBP(%rsp), %rbp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	movq R9(%rsp), %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	movq R8(%rsp), %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	movq RDI(%rsp), %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	movq RSI(%rsp), %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	movq RDX(%rsp), %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	movq RCX(%rsp), %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	movq RAX(%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	addq $MCOUNT_REG_SIZE-\save, %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #ifdef CONFIG_DYNAMIC_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) SYM_FUNC_START(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	retq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) SYM_FUNC_END(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) EXPORT_SYMBOL(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) SYM_FUNC_START(ftrace_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	/* save_mcount_regs fills in first two parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	save_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) SYM_INNER_LABEL(ftrace_caller_op_ptr, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	/* Load the ftrace_ops into the 3rd parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	movq function_trace_op(%rip), %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	/* regs go into 4th parameter (but make it NULL) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	movq $0, %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	call ftrace_stub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	restore_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	 * The code up to this label is copied into trampolines so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	 * think twice before adding any new code or changing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	 * layout here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) SYM_INNER_LABEL(ftrace_caller_end, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	jmp ftrace_epilogue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) SYM_FUNC_END(ftrace_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) SYM_FUNC_START(ftrace_epilogue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	jmp ftrace_stub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)  * This is weak to keep gas from relaxing the jumps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)  * It is also used to copy the retq for trampolines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	retq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) SYM_FUNC_END(ftrace_epilogue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) SYM_FUNC_START(ftrace_regs_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	/* Save the current flags before any operations that can change them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	pushfq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	/* added 8 bytes to save flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	save_mcount_regs 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	/* save_mcount_regs fills in first two parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) SYM_INNER_LABEL(ftrace_regs_caller_op_ptr, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	/* Load the ftrace_ops into the 3rd parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	movq function_trace_op(%rip), %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	/* Save the rest of pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	movq %r15, R15(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	movq %r14, R14(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	movq %r13, R13(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	movq %r12, R12(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	movq %r11, R11(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	movq %r10, R10(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	movq %rbx, RBX(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	/* Copy saved flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	movq MCOUNT_REG_SIZE(%rsp), %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	movq %rcx, EFLAGS(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	/* Kernel segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	movq $__KERNEL_DS, %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	movq %rcx, SS(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	movq $__KERNEL_CS, %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	movq %rcx, CS(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	/* Stack - skipping return address and flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	movq %rcx, RSP(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	ENCODE_FRAME_POINTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	/* regs go into 4th parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	leaq (%rsp), %rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	call ftrace_stub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	/* Copy flags back to SS, to restore them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	movq EFLAGS(%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	movq %rax, MCOUNT_REG_SIZE(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	/* Handlers can change the RIP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	movq RIP(%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	movq %rax, MCOUNT_REG_SIZE+8(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	/* restore the rest of pt_regs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	movq R15(%rsp), %r15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	movq R14(%rsp), %r14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	movq R13(%rsp), %r13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	movq R12(%rsp), %r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	movq R10(%rsp), %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	movq RBX(%rsp), %rbx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	movq ORIG_RAX(%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	movq %rax, MCOUNT_REG_SIZE-8(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	 * If ORIG_RAX is anything but zero, make this a call to that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	 * See arch_ftrace_set_direct_caller().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	movq ORIG_RAX(%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	testq	%rax, %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) SYM_INNER_LABEL(ftrace_regs_caller_jmp, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	jnz	1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	restore_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	/* Restore flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	popfq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	 * As this jmp to ftrace_epilogue can be a short jump
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	 * it must not be copied into the trampoline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	 * The trampoline will add the code to jump
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	 * to the return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	jmp ftrace_epilogue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	/* Swap the flags with orig_rax */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 1:	movq MCOUNT_REG_SIZE(%rsp), %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	movq %rdi, MCOUNT_REG_SIZE-8(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	movq %rax, MCOUNT_REG_SIZE(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	restore_mcount_regs 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	/* Restore flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	popfq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	UNWIND_HINT_RET_OFFSET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	jmp	ftrace_epilogue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) SYM_FUNC_END(ftrace_regs_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #else /* ! CONFIG_DYNAMIC_FTRACE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) SYM_FUNC_START(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	cmpq $ftrace_stub, ftrace_trace_function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	jnz trace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) fgraph_trace:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	cmpq $ftrace_stub, ftrace_graph_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	jnz ftrace_graph_caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	jnz ftrace_graph_caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) SYM_INNER_LABEL(ftrace_stub, SYM_L_GLOBAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	retq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) trace:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	/* save_mcount_regs fills in first two parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	save_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	 * set (see include/asm/ftrace.h and include/linux/ftrace.h).  Only the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	 * ip and parent ip are used and the list function is called when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	 * function tracing is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	movq ftrace_trace_function, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	CALL_NOSPEC r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	restore_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	jmp fgraph_trace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) SYM_FUNC_END(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) EXPORT_SYMBOL(__fentry__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) #endif /* CONFIG_DYNAMIC_FTRACE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #ifdef CONFIG_FUNCTION_GRAPH_TRACER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) SYM_FUNC_START(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	/* Saves rbp into %rdx and fills first parameter  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	save_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	movq $0, %rdx	/* No framepointers needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	call	prepare_ftrace_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	restore_mcount_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	retq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) SYM_FUNC_END(ftrace_graph_caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) SYM_CODE_START(return_to_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	UNWIND_HINT_EMPTY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	subq  $24, %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	/* Save the return values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	movq %rax, (%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	movq %rdx, 8(%rsp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	movq %rbp, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	call ftrace_return_to_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	movq %rax, %rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	movq 8(%rsp), %rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	movq (%rsp), %rax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	addq $24, %rsp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	JMP_NOSPEC rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) SYM_CODE_END(return_to_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) #endif