Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *  Kernel Probes (KProbes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) IBM Corporation, 2002, 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *		Probes initial implementation ( includes contributions from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *		Rusty Russell).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *		interface to access function arguments.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * 2004-Nov	Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  *		for PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/extable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <asm/code-patching.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <asm/sstep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <asm/inst.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) bool arch_within_kprobe_blacklist(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	return  (addr >= (unsigned long)__kprobes_text_start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		 addr < (unsigned long)__kprobes_text_end) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 		(addr >= (unsigned long)_stext &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 		 addr < (unsigned long)__head_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	kprobe_opcode_t *addr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #ifdef PPC64_ELF_ABI_v2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	/* PPC64 ABIv2 needs local entry point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	if (addr && !offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #ifdef CONFIG_KPROBES_ON_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 		unsigned long faddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 		 * Per livepatch.h, ftrace location is always within the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		 * 16 bytes of a function on powerpc with -mprofile-kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 		faddr = ftrace_location_range((unsigned long)addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 					      (unsigned long)addr + 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		if (faddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 			addr = (kprobe_opcode_t *)faddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 			addr = (kprobe_opcode_t *)ppc_function_entry(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #elif defined(PPC64_ELF_ABI_v1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	 * 64bit powerpc ABIv1 uses function descriptors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	 * - Check for the dot variant of the symbol first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	 * - If that fails, try looking up the symbol provided.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	 * This ensures we always get to the actual symbol and not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	 * the descriptor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	 * Also handle <module:symbol> format.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	bool dot_appended = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	const char *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	ssize_t ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		c++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		len = c - name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 		memcpy(dot_name, name, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	} else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		c = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	if (*c != '\0' && *c != '.') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		dot_name[len++] = '.';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		dot_appended = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	/* Fallback to the original non-dot symbol lookup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	if (!addr && dot_appended)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int arch_prepare_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	struct kprobe *prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	if ((unsigned long)p->addr & 0x03) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		printk("Attempt to register kprobe at an unaligned address\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	} else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	} else if ((unsigned long)p->addr & ~PAGE_MASK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		   ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)(p->addr - 1)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		printk("Cannot register a kprobe on the second word of prefixed instruction\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	prev = get_kprobe(p->addr - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	if (prev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	    ppc_inst_prefixed(ppc_inst_read((struct ppc_inst *)prev->ainsn.insn))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		printk("Cannot register a kprobe on the second word of prefixed instruction\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	/* insn must be on a special executable page on ppc64.  This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	 * not explicitly required on ppc32 (right now), but it doesn't hurt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		p->ainsn.insn = get_insn_slot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		if (!p->ainsn.insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 			ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 		patch_instruction((struct ppc_inst *)p->ainsn.insn, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		p->opcode = ppc_inst_val(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	p->ainsn.boostable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) NOKPROBE_SYMBOL(arch_prepare_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) void arch_arm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	patch_instruction((struct ppc_inst *)p->addr, ppc_inst(BREAKPOINT_INSTRUCTION));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) NOKPROBE_SYMBOL(arch_arm_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) void arch_disarm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	patch_instruction((struct ppc_inst *)p->addr, ppc_inst(p->opcode));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) NOKPROBE_SYMBOL(arch_disarm_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) void arch_remove_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	if (p->ainsn.insn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 		free_insn_slot(p->ainsn.insn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		p->ainsn.insn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) NOKPROBE_SYMBOL(arch_remove_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	enable_single_step(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	 * On powerpc we should single step on the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	 * instruction even if the probed insn is a trap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	 * variant as values in regs could play a part in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	 * if the trap is taken or not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	regs->nip = (unsigned long)p->ainsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	kcb->prev_kprobe.kp = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	kcb->prev_kprobe.status = kcb->kprobe_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	kcb->kprobe_status = kcb->prev_kprobe.status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 				struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	__this_cpu_write(current_kprobe, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	kcb->kprobe_saved_msr = regs->msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) bool arch_kprobe_on_func_entry(unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #ifdef PPC64_ELF_ABI_v2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #ifdef CONFIG_KPROBES_ON_FTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	return offset <= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	return offset <= 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	return !offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	ri->ret_addr = (kprobe_opcode_t *)regs->link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	ri->fp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	/* Replace the return addr with trampoline addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	regs->link = (unsigned long)kretprobe_trampoline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) NOKPROBE_SYMBOL(arch_prepare_kretprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	struct ppc_inst insn = ppc_inst_read((struct ppc_inst *)p->ainsn.insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	/* regs->nip is also adjusted if emulate_step returns 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	ret = emulate_step(regs, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		 * Once this instruction has been boosted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 		 * successfully, set the boostable flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		if (unlikely(p->ainsn.boostable == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			p->ainsn.boostable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	} else if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		 * So, we should never get here... but, its still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 		 * good to catch them, just in case...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 		printk("Can't step on instruction %s\n", ppc_inst_as_str(insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 		BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		 * If we haven't previously emulated this instruction, then it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 		 * can't be boosted. Note it down so we don't try to do so again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		 * If, however, we had emulated this instruction in the past,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		 * then this is just an error with the current run (for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 		 * instance, exceptions due to a load/store). We return 0 so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 		 * that this is now single-stepped, but continue to try
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		 * emulating it in subsequent probe hits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		if (unlikely(p->ainsn.boostable != 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 			p->ainsn.boostable = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) NOKPROBE_SYMBOL(try_to_emulate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) int kprobe_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	struct kprobe *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	unsigned int *addr = (unsigned int *)regs->nip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	struct kprobe_ctlblk *kcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	if (!IS_ENABLED(CONFIG_BOOKE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	    (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	 * We don't want to be preempted for the entire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	 * duration of kprobe processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	p = get_kprobe(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	if (!p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		unsigned int instr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		if (get_kernel_nofault(instr, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 			goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 		if (instr != BREAKPOINT_INSTRUCTION) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 			 * PowerPC has multiple variants of the "trap"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 			 * instruction. If the current instruction is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 			 * trap variant, it could belong to someone else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 			if (is_trap(instr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 				goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 			 * The breakpoint instruction was removed right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 			 * after we hit it.  Another cpu has removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 			 * either a probepoint or a debugger breakpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 			 * at this address.  In either case, no further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 			 * handling of this interrupt is appropriate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 			ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 		/* Not one of ours: let kernel handle it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	/* Check we're not actually recursing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	if (kprobe_running()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 		kprobe_opcode_t insn = *p->ainsn.insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 			/* Turn off 'trace' bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 			regs->msr &= ~MSR_SINGLESTEP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 			regs->msr |= kcb->kprobe_saved_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 			goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 		 * We have reentered the kprobe_handler(), since another probe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 		 * was hit while within the handler. We here save the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 		 * kprobes variables and just single step on the instruction of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 		 * the new probe without calling any user handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 		save_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		set_current_kprobe(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		kprobes_inc_nmissed_count(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 		kcb->kprobe_status = KPROBE_REENTER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 		if (p->ainsn.boostable >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 			ret = try_to_emulate(p, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 			if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 				restore_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 				preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 				return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 		prepare_singlestep(p, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	set_current_kprobe(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	if (p->pre_handler && p->pre_handler(p, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 		/* handler changed execution path, so skip ss setup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 		reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	if (p->ainsn.boostable >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 		ret = try_to_emulate(p, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 		if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 			if (p->post_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 				p->post_handler(p, regs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 			kcb->kprobe_status = KPROBE_HIT_SSDONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 			reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 			preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	prepare_singlestep(p, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	kcb->kprobe_status = KPROBE_HIT_SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) no_kprobe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) NOKPROBE_SYMBOL(kprobe_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)  * Function return probe trampoline:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)  * 	- init_kprobes() establishes a probepoint here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)  * 	- When the probed function returns, this probe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)  * 		causes the handlers to fire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) asm(".global kretprobe_trampoline\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	".type kretprobe_trampoline, @function\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	"kretprobe_trampoline:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	"nop\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	"blr\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	".size kretprobe_trampoline, .-kretprobe_trampoline\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)  * Called when the probe at kretprobe trampoline is hit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	unsigned long orig_ret_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 	 * We get here through one of two paths:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 	 * 1. by taking a trap -> kprobe_handler() -> here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	 * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	 * When going back through (1), we need regs->nip to be setup properly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	 * as it is used to determine the return address from the trap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 	 * For (2), since nip is not honoured with optprobes, we instead setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 	 * the link register properly so that the subsequent 'blr' in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	 * kretprobe_trampoline jumps back to the right instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	 * For nip, we should set the address to the previous instruction since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	 * we end up emulating it in kprobe_handler(), which increments the nip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	 * again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	regs->nip = orig_ret_address - 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	regs->link = orig_ret_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) NOKPROBE_SYMBOL(trampoline_probe_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)  * Called after single-stepping.  p->addr is the address of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)  * instruction whose first byte has been replaced by the "breakpoint"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)  * instruction.  To avoid the SMP problems that can occur when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)  * temporarily put back the original opcode to single-step, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)  * single-stepped a copy of the instruction.  The address of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)  * copy is p->ainsn.insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) int kprobe_post_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 	int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	struct kprobe *cur = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 	if (!cur || user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 	len = ppc_inst_len(ppc_inst_read((struct ppc_inst *)cur->ainsn.insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	/* make sure we got here for instruction we have a kprobe on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 		cur->post_handler(cur, regs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	/* Adjust nip to after the single-stepped instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	regs->nip = (unsigned long)cur->addr + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 	regs->msr |= kcb->kprobe_saved_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	/*Restore back the original saved kprobes variables and continue. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	if (kcb->kprobe_status == KPROBE_REENTER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 		restore_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 	preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	 * if somebody else is singlestepping across a probe point, msr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	 * will have DE/SE set, in which case, continue the remaining processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	 * of do_debug, as if this is not a probe hit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 	if (regs->msr & MSR_SINGLESTEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) NOKPROBE_SYMBOL(kprobe_post_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 	struct kprobe *cur = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	const struct exception_table_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 	switch(kcb->kprobe_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 	case KPROBE_HIT_SS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 	case KPROBE_REENTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 		 * We are here because the instruction being single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 		 * stepped caused a page fault. We reset the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 		 * kprobe and the nip points back to the probe address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 		 * and allow the page fault handler to continue as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 		 * normal page fault.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 		regs->nip = (unsigned long)cur->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 		regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 		regs->msr |= kcb->kprobe_saved_msr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 		if (kcb->kprobe_status == KPROBE_REENTER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 			restore_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 			reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 		preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 	case KPROBE_HIT_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	case KPROBE_HIT_SSDONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 		 * We increment the nmissed count for accounting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 		 * we can also use npre/npostfault count for accounting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) 		 * these specific fault cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 		kprobes_inc_nmissed_count(cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 		 * We come here because instructions in the pre/post
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 		 * handler caused the page_fault, this could happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) 		 * if handler tries to access user space by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 		 * copy_from_user(), get_user() etc. Let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 		 * user-specified handler try to fix it first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 		 * In case the user-specified fault handler returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) 		 * zero, try to fix up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) 		if ((entry = search_exception_tables(regs->nip)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) 			regs->nip = extable_fixup(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 		 * fixup_exception() could not handle it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 		 * Let do_page_fault() fix it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) 	default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) NOKPROBE_SYMBOL(kprobe_fault_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) unsigned long arch_deref_entry_point(void *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) #ifdef PPC64_ELF_ABI_v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) 	if (!kernel_text_address((unsigned long)entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) 		return ppc_global_function_entry(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) 		return (unsigned long)entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) NOKPROBE_SYMBOL(arch_deref_entry_point);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) static struct kprobe trampoline_p = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) 	.addr = (kprobe_opcode_t *) &kretprobe_trampoline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) 	.pre_handler = trampoline_probe_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) int __init arch_init_kprobes(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 	return register_kprobe(&trampoline_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) int arch_trampoline_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) 	if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) NOKPROBE_SYMBOL(arch_trampoline_kprobe);