^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel Probes Jump Optimization (Optprobes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) IBM Corporation, 2002, 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) Hitachi Ltd., 2012
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/objtool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/static_call.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/text-patching.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/desc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/debugreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/nospec-branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include "common.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) unsigned long __recover_optprobed_insn(kprobe_opcode_t *buf, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct optimized_kprobe *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct kprobe *kp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) long offs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) for (i = 0; i < JMP32_INSN_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) kp = get_kprobe((void *)addr - i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /* This function only handles jump-optimized kprobe */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (kp && kprobe_optimized(kp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) op = container_of(kp, struct optimized_kprobe, kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /* If op->list is not empty, op is under optimizing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) if (list_empty(&op->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * If the kprobe can be optimized, original bytes which can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * overwritten by jump destination address. In this case, original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * bytes must be recovered from op->optinsn.copied_insn buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if (copy_from_kernel_nofault(buf, (void *)addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return 0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (addr == (unsigned long)kp->addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) buf[0] = kp->opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) memcpy(buf + 1, op->optinsn.copied_insn, DISP32_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) offs = addr - (unsigned long)kp->addr - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) memcpy(buf, op->optinsn.copied_insn + offs, DISP32_SIZE - offs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return (unsigned long)buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static void synthesize_clac(kprobe_opcode_t *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * Can't be static_cpu_has() due to how objtool treats this feature bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * This isn't a fast path anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (!boot_cpu_has(X86_FEATURE_SMAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /* Replace the NOP3 with CLAC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) addr[0] = 0x0f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) addr[1] = 0x01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) addr[2] = 0xca;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) *addr++ = 0x48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) *addr++ = 0xbf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) *addr++ = 0xb8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *(unsigned long *)addr = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) asm (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ".pushsection .rodata\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) "optprobe_template_func:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) ".pushsection .discard.func_stack_frame_non_standard\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) "__func_stack_frame_non_standard_optprobe_template_func:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ".quad optprobe_template_func\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ".long optprobe_template_func\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) ".popsection\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ".global optprobe_template_entry\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) "optprobe_template_entry:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* We don't bother saving the ss register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) " pushq %rsp\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) " pushfq\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) ".global optprobe_template_clac\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) "optprobe_template_clac:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) ASM_NOP3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) SAVE_REGS_STRING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) " movq %rsp, %rsi\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) ".global optprobe_template_val\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) "optprobe_template_val:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) ASM_NOP5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) ASM_NOP5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) ".global optprobe_template_call\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "optprobe_template_call:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ASM_NOP5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) /* Move flags to rsp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) " movq 18*8(%rsp), %rdx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) " movq %rdx, 19*8(%rsp)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) RESTORE_REGS_STRING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* Skip flags entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) " addq $8, %rsp\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) " popfq\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #else /* CONFIG_X86_32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) " pushl %esp\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) " pushfl\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) ".global optprobe_template_clac\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) "optprobe_template_clac:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) ASM_NOP3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) SAVE_REGS_STRING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) " movl %esp, %edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) ".global optprobe_template_val\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) "optprobe_template_val:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ASM_NOP5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) ".global optprobe_template_call\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) "optprobe_template_call:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) ASM_NOP5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /* Move flags into esp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) " movl 14*4(%esp), %edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) " movl %edx, 15*4(%esp)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) RESTORE_REGS_STRING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /* Skip flags entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) " addl $4, %esp\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) " popfl\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ".global optprobe_template_end\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) "optprobe_template_end:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) ".popsection\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define TMPL_CLAC_IDX \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) ((long)optprobe_template_clac - (long)optprobe_template_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define TMPL_MOVE_IDX \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) ((long)optprobe_template_val - (long)optprobe_template_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define TMPL_CALL_IDX \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ((long)optprobe_template_call - (long)optprobe_template_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #define TMPL_END_IDX \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) ((long)optprobe_template_end - (long)optprobe_template_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Optimized kprobe call back function: called from optinsn */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* This is possible if op is under delayed unoptimizing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (kprobe_disabled(&op->kp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (kprobe_running()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) kprobes_inc_nmissed_count(&op->kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /* Save skipped registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) regs->cs = __KERNEL_CS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) regs->gs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) regs->ip = (unsigned long)op->kp.addr + INT3_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) regs->orig_ax = ~0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) __this_cpu_write(current_kprobe, &op->kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) kcb->kprobe_status = KPROBE_HIT_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) opt_pre_handler(&op->kp, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) __this_cpu_write(current_kprobe, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) NOKPROBE_SYMBOL(optimized_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct insn insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int len = 0, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) while (len < JMP32_INSN_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) ret = __copy_instruction(dest + len, src + len, real + len, &insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (!ret || !can_boost(&insn, src + len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) len += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /* Check whether the address range is reserved */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (ftrace_text_reserved(src, src + len - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) alternatives_text_reserved(src, src + len - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) jump_label_text_reserved(src, src + len - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static_call_text_reserved(src, src + len - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* Check whether insn is indirect jump */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static int __insn_is_indirect_jump(struct insn *insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return ((insn->opcode.bytes[0] == 0xff &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) insn->opcode.bytes[0] == 0xea); /* Segment based jump */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* Check whether insn jumps into specified address range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) unsigned long target = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) switch (insn->opcode.bytes[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) case 0xe0: /* loopne */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) case 0xe1: /* loope */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) case 0xe2: /* loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) case 0xe3: /* jcxz */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) case 0xe9: /* near relative jump */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) case 0xeb: /* short relative jump */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) case 0x0f:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) target = (unsigned long)insn->next_byte + insn->immediate.value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return (start <= target && target <= start + len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static int insn_is_indirect_jump(struct insn *insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) int ret = __insn_is_indirect_jump(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #ifdef CONFIG_RETPOLINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * Jump to x86_indirect_thunk_* is treated as an indirect jump.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * Note that even with CONFIG_RETPOLINE=y, the kernel compiled with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * older gcc may use indirect jump. So we add this check instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * replace indirect-jump check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) ret = insn_jump_into_range(insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) (unsigned long)__indirect_thunk_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) (unsigned long)__indirect_thunk_end -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) (unsigned long)__indirect_thunk_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) static bool is_padding_int3(unsigned long addr, unsigned long eaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) unsigned char ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) for (; addr < eaddr; addr++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (get_kernel_nofault(ops, (void *)addr) < 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) ops != INT3_INSN_OPCODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* Decode whole function to ensure any instructions don't jump into target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static int can_optimize(unsigned long paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) unsigned long addr, size = 0, offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct insn insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) kprobe_opcode_t buf[MAX_INSN_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Lookup symbol including addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (!kallsyms_lookup_size_offset(paddr, &size, &offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * Do not optimize in the entry code due to the unstable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * stack handling and registers setup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (((paddr >= (unsigned long)__entry_text_start) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) (paddr < (unsigned long)__entry_text_end)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* Check there is enough space for a relative jump. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (size - offset < JMP32_INSN_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /* Decode instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) addr = paddr - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) while (addr < paddr - offset + size) { /* Decode until function end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) unsigned long recovered_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (search_exception_tables(addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Since some fixup code will jumps into this function,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * we can't optimize kprobe in this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) recovered_insn = recover_probed_instruction(buf, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!recovered_insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) insn_get_length(&insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * In the case of detecting unknown breakpoint, this could be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * a padding INT3 between functions. Let's check that all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * rest of the bytes are also INT3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (insn.opcode.bytes[0] == INT3_INSN_OPCODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return is_padding_int3(addr, paddr - offset + size) ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /* Recover address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) insn.kaddr = (void *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) insn.next_byte = (void *)(addr + insn.length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) /* Check any instructions don't jump into target */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (insn_is_indirect_jump(&insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) insn_jump_into_range(&insn, paddr + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) DISP32_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) addr += insn.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /* Check optimized_kprobe can actually be optimized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) int arch_check_optimized_kprobe(struct optimized_kprobe *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct kprobe *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) for (i = 1; i < op->optinsn.size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) p = get_kprobe(op->kp.addr + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (p && !kprobe_disabled(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /* Check the addr is within the optimized instructions. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) int arch_within_optimized_kprobe(struct optimized_kprobe *op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return ((unsigned long)op->kp.addr <= addr &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) (unsigned long)op->kp.addr + op->optinsn.size > addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /* Free optimized instruction slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) u8 *slot = op->optinsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (slot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) int len = TMPL_END_IDX + op->optinsn.size + JMP32_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /* Record the perf event before freeing the slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) perf_event_text_poke(slot, slot, len, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) free_optinsn_slot(slot, dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) op->optinsn.insn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) op->optinsn.size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) __arch_remove_optimized_kprobe(op, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * Copy replacing target instructions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * Target instructions MUST be relocatable (checked inside)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * This is called when new aggr(opt)probe is allocated or reused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) int arch_prepare_optimized_kprobe(struct optimized_kprobe *op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct kprobe *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) u8 *buf = NULL, *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) int ret, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) long rel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (!can_optimize((unsigned long)op->kp.addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) return -EILSEQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) buf = kzalloc(MAX_OPTINSN_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) op->optinsn.insn = slot = get_optinsn_slot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (!slot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Verify if the address gap is in 2GB range, because this uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * a relative jump.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) rel = (long)slot - (long)op->kp.addr + JMP32_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (abs(rel) > 0x7fffffff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) ret = -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /* Copy arch-dep-instance from template */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) memcpy(buf, optprobe_template_entry, TMPL_END_IDX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) /* Copy instructions into the out-of-line buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) slot + TMPL_END_IDX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) op->optinsn.size = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) len = TMPL_END_IDX + op->optinsn.size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) synthesize_clac(buf + TMPL_CLAC_IDX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /* Set probe information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Set probe function call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) synthesize_relcall(buf + TMPL_CALL_IDX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) slot + TMPL_CALL_IDX, optimized_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /* Set returning jmp instruction at the tail of out-of-line buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) synthesize_reljump(buf + len, slot + len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) (u8 *)op->kp.addr + op->optinsn.size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) len += JMP32_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * Note len = TMPL_END_IDX + op->optinsn.size + JMP32_INSN_SIZE is also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * used in __arch_remove_optimized_kprobe().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* We have to use text_poke() for instruction buffer because it is RO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) perf_event_text_poke(slot, NULL, 0, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) text_poke(slot, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) __arch_remove_optimized_kprobe(op, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * Replace breakpoints (INT3) with relative jumps (JMP.d32).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * Caller must call with locking kprobe_mutex and text_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * The caller will have installed a regular kprobe and after that issued
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * syncrhonize_rcu_tasks(), this ensures that the instruction(s) that live in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * the 4 bytes after the INT3 are unused and can now be overwritten.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) void arch_optimize_kprobes(struct list_head *oplist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct optimized_kprobe *op, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) u8 insn_buff[JMP32_INSN_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) list_for_each_entry_safe(op, tmp, oplist, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) s32 rel = (s32)((long)op->optinsn.insn -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) ((long)op->kp.addr + JMP32_INSN_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) WARN_ON(kprobe_disabled(&op->kp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /* Backup instructions which will be replaced by jump address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) DISP32_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) insn_buff[0] = JMP32_INSN_OPCODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) *(s32 *)(&insn_buff[1]) = rel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) text_poke_bp(op->kp.addr, insn_buff, JMP32_INSN_SIZE, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) list_del_init(&op->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * Replace a relative jump (JMP.d32) with a breakpoint (INT3).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * After that, we can restore the 4 bytes after the INT3 to undo what
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * arch_optimize_kprobes() scribbled. This is safe since those bytes will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * unused once the INT3 lands.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) void arch_unoptimize_kprobe(struct optimized_kprobe *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) u8 new[JMP32_INSN_SIZE] = { INT3_INSN_OPCODE, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) u8 old[JMP32_INSN_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) u8 *addr = op->kp.addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) memcpy(old, op->kp.addr, JMP32_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) memcpy(new + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) op->optinsn.copied_insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) JMP32_INSN_SIZE - INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) text_poke(addr, new, INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) text_poke_sync();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) text_poke(addr + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) new + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) JMP32_INSN_SIZE - INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) text_poke_sync();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) perf_event_text_poke(op->kp.addr, old, JMP32_INSN_SIZE, new, JMP32_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * Recover original instructions and breakpoints from relative jumps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * Caller must call with locking kprobe_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) extern void arch_unoptimize_kprobes(struct list_head *oplist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct list_head *done_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct optimized_kprobe *op, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) list_for_each_entry_safe(op, tmp, oplist, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) arch_unoptimize_kprobe(op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) list_move(&op->list, done_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) struct optimized_kprobe *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (p->flags & KPROBE_FLAG_OPTIMIZED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) /* This kprobe is really able to run optimized path. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) op = container_of(p, struct optimized_kprobe, kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /* Detour through copied instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (!reenter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) NOKPROBE_SYMBOL(setup_detour_execution);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags