^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Kernel Probes (KProbes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corp. 2002, 2006
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/moduleloader.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/dis.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "entry.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) DEFINE_PER_CPU(struct kprobe *, current_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct kretprobe_blackpoint kretprobe_blacklist[] = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) DEFINE_INSN_CACHE_OPS(s390_insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static int insn_page_in_use;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) void *alloc_insn_page(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) void *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) page = module_alloc(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) __set_memory((unsigned long) page, 1, SET_MEMORY_RO | SET_MEMORY_X);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) void free_insn_page(void *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) module_memfree(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) static void *alloc_s390_insn_page(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) if (xchg(&insn_page_in_use, 1) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return &kprobes_insn_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static void free_s390_insn_page(void *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) xchg(&insn_page_in_use, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct kprobe_insn_cache kprobe_s390_insn_slots = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) .alloc = alloc_s390_insn_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) .free = free_s390_insn_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) .insn_size = MAX_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static void copy_instruction(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) kprobe_opcode_t insn[MAX_INSN_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) s64 disp, new_disp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) u64 addr, new_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) len = insn_length(*p->addr >> 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) memcpy(&insn, p->addr, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) p->opcode = insn[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (probe_is_insn_relative_long(&insn[0])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * For pc-relative instructions in RIL-b or RIL-c format patch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * the RI2 displacement field. We have already made sure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * the insn slot for the patched instruction is within the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * 2GB area as the original instruction (either kernel image or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * module area). Therefore the new displacement will always fit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) disp = *(s32 *)&insn[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) addr = (u64)(unsigned long)p->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) new_addr = (u64)(unsigned long)p->ainsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) new_disp = ((addr + (disp * 2)) - new_addr) / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *(s32 *)&insn[1] = new_disp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) s390_kernel_write(p->ainsn.insn, &insn, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) NOKPROBE_SYMBOL(copy_instruction);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static inline int is_kernel_addr(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return addr < (void *)_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static int s390_get_insn_slot(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * Get an insn slot that is within the same 2GB area like the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * instruction. That way instructions with a 32bit signed displacement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * field can be patched and executed within the insn slot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) p->ainsn.insn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (is_kernel_addr(p->addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) p->ainsn.insn = get_s390_insn_slot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) else if (is_module_addr(p->addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) p->ainsn.insn = get_insn_slot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) return p->ainsn.insn ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) NOKPROBE_SYMBOL(s390_get_insn_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void s390_free_insn_slot(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (!p->ainsn.insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (is_kernel_addr(p->addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) free_s390_insn_slot(p->ainsn.insn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) free_insn_slot(p->ainsn.insn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) p->ainsn.insn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) NOKPROBE_SYMBOL(s390_free_insn_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int arch_prepare_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if ((unsigned long) p->addr & 0x01)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Make sure the probe isn't going on a difficult instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (probe_is_prohibited_opcode(p->addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (s390_get_insn_slot(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) copy_instruction(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) NOKPROBE_SYMBOL(arch_prepare_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct swap_insn_args {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct kprobe *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned int arm_kprobe : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static int swap_instruction(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct swap_insn_args *args = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct kprobe *p = args->p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) u16 opc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) s390_kernel_write(p->addr, &opc, sizeof(opc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) NOKPROBE_SYMBOL(swap_instruction);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) void arch_arm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) stop_machine_cpuslocked(swap_instruction, &args, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) NOKPROBE_SYMBOL(arch_arm_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) void arch_disarm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) stop_machine_cpuslocked(swap_instruction, &args, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) NOKPROBE_SYMBOL(arch_disarm_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void arch_remove_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) s390_free_insn_slot(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) NOKPROBE_SYMBOL(arch_remove_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void enable_singlestep(struct kprobe_ctlblk *kcb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned long ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct per_regs per_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* Set up the PER control registers %cr9-%cr11 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) per_kprobe.control = PER_EVENT_IFETCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) per_kprobe.start = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) per_kprobe.end = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* Save control regs and psw mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) kcb->kprobe_saved_imask = regs->psw.mask &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /* Set PER control regs, turns on single step for the given address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) __ctl_load(per_kprobe, 9, 11);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) regs->psw.mask |= PSW_MASK_PER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) regs->psw.addr = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) NOKPROBE_SYMBOL(enable_singlestep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static void disable_singlestep(struct kprobe_ctlblk *kcb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) unsigned long ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* Restore control regs and psw mask, set new psw address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) regs->psw.mask &= ~PSW_MASK_PER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) regs->psw.mask |= kcb->kprobe_saved_imask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) regs->psw.addr = ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) NOKPROBE_SYMBOL(disable_singlestep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * Activate a kprobe by storing its pointer to current_kprobe. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * two kprobes can be active, see KPROBE_REENTER.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) kcb->prev_kprobe.status = kcb->kprobe_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) __this_cpu_write(current_kprobe, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) NOKPROBE_SYMBOL(push_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * Deactivate a kprobe by backing up to the previous state. If the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * for any other state prev_kprobe.kp will be NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static void pop_kprobe(struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) kcb->kprobe_status = kcb->prev_kprobe.status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) NOKPROBE_SYMBOL(pop_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) ri->fp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /* Replace the return addr with trampoline addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) NOKPROBE_SYMBOL(arch_prepare_kretprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) switch (kcb->kprobe_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) case KPROBE_HIT_SSDONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) case KPROBE_HIT_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) kprobes_inc_nmissed_count(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) case KPROBE_HIT_SS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) case KPROBE_REENTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * A kprobe on the code path to single step an instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * is a BUG. The code path resides in the .kprobes.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * section and is executed with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) pr_err("Invalid kprobe detected.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) dump_kprobe(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) NOKPROBE_SYMBOL(kprobe_reenter_check);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static int kprobe_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct kprobe_ctlblk *kcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) struct kprobe *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * We want to disable preemption for the entire duration of kprobe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * processing. That includes the calls to the pre/post handlers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * and single stepping the kprobe instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) p = get_kprobe((void *)(regs->psw.addr - 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (kprobe_running()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * We have hit a kprobe while another is still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * active. This can happen in the pre and post
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * handler. Single step the instruction of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * new probe but do not call any handler function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * of this secondary kprobe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * push_kprobe and pop_kprobe saves and restores
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * the currently active kprobe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) kprobe_reenter_check(kcb, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) push_kprobe(kcb, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) kcb->kprobe_status = KPROBE_REENTER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * If we have no pre-handler or it returned 0, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * continue with single stepping. If we have a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * pre-handler and it returned non-zero, it prepped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * for changing execution path, so get out doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * nothing more here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) push_kprobe(kcb, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) kcb->kprobe_status = KPROBE_HIT_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (p->pre_handler && p->pre_handler(p, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) pop_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) kcb->kprobe_status = KPROBE_HIT_SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) } /* else:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * No kprobe at this address and no active kprobe. The trap has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * not been caused by a kprobe breakpoint. The race of breakpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * vs. kprobe remove does not exist because on s390 as we use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * stop_machine to arm/disarm the breakpoints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) NOKPROBE_SYMBOL(kprobe_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) * Function return probe trampoline:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * - init_kprobes() establishes a probepoint here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * - When the probed function returns, this probe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * causes the handlers to fire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) static void __used kretprobe_trampoline_holder(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) asm volatile(".global kretprobe_trampoline\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) "kretprobe_trampoline: bcr 0,0\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * Called when the probe at kretprobe trampoline is hit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) regs->psw.addr = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * By returning a non-zero value, we are telling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * kprobe_handler() that we don't want the post_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * to run (and have re-enabled preemption)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) NOKPROBE_SYMBOL(trampoline_probe_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * Called after single-stepping. p->addr is the address of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * instruction whose first byte has been replaced by the "breakpoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * instruction. To avoid the SMP problems that can occur when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * temporarily put back the original opcode to single-step, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * single-stepped a copy of the instruction. The address of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * copy is p->ainsn.insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static void resume_execution(struct kprobe *p, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) unsigned long ip = regs->psw.addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) int fixup = probe_get_fixup_type(p->ainsn.insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (fixup & FIXUP_PSW_NORMAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) int ilen = insn_length(p->ainsn.insn[0] >> 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (ip - (unsigned long) p->ainsn.insn == ilen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) ip = (unsigned long) p->addr + ilen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (fixup & FIXUP_RETURN_REGISTER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) regs->gprs[reg] += (unsigned long) p->addr -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) (unsigned long) p->ainsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) disable_singlestep(kcb, regs, ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) NOKPROBE_SYMBOL(resume_execution);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) static int post_kprobe_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) struct kprobe *p = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) kcb->kprobe_status = KPROBE_HIT_SSDONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) p->post_handler(p, regs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) resume_execution(p, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) pop_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * if somebody else is singlestepping across a probe point, psw mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * will have PER set, in which case, continue the remaining processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * of do_single_step, as if this is not a probe hit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (regs->psw.mask & PSW_MASK_PER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) NOKPROBE_SYMBOL(post_kprobe_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct kprobe *p = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) const struct exception_table_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) switch(kcb->kprobe_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) case KPROBE_HIT_SS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) case KPROBE_REENTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * We are here because the instruction being single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * stepped caused a page fault. We reset the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * kprobe and the nip points back to the probe address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * and allow the page fault handler to continue as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * normal page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) disable_singlestep(kcb, regs, (unsigned long) p->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) pop_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) case KPROBE_HIT_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) case KPROBE_HIT_SSDONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * We increment the nmissed count for accounting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * we can also use npre/npostfault count for accounting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * these specific fault cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) kprobes_inc_nmissed_count(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * We come here because instructions in the pre/post
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * handler caused the page_fault, this could happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * if handler tries to access user space by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * copy_from_user(), get_user() etc. Let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * user-specified handler try to fix it first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (p->fault_handler && p->fault_handler(p, regs, trapnr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * In case the user-specified fault handler returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * zero, try to fix up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) entry = s390_search_extables(regs->psw.addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (entry && ex_handle(entry, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * fixup_exception() could not handle it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * Let do_page_fault() fix it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) NOKPROBE_SYMBOL(kprobe_trap_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) ret = kprobe_trap_handler(regs, trapnr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) NOKPROBE_SYMBOL(kprobe_fault_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Wrapper routine to for handling exceptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) int kprobe_exceptions_notify(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) unsigned long val, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct die_args *args = (struct die_args *) data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct pt_regs *regs = args->regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) int ret = NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) switch (val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) case DIE_BPT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (kprobe_handler(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) case DIE_SSTEP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (post_kprobe_handler(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) case DIE_TRAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!preemptible() && kprobe_running() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) kprobe_trap_handler(regs, args->trapnr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) NOKPROBE_SYMBOL(kprobe_exceptions_notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) static struct kprobe trampoline = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) .pre_handler = trampoline_probe_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) int __init arch_init_kprobes(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return register_kprobe(&trampoline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int arch_trampoline_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) NOKPROBE_SYMBOL(arch_trampoline_kprobe);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags