Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *  Kernel Probes (KProbes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *  arch/mips/kernel/kprobes.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *  Copyright 2006 Sony Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *  Copyright 2010 Cavium Networks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *  Some portions copied from the powerpc version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *   Copyright (C) IBM Corporation, 2002, 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <asm/branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <asm/break.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include "probes-common.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) static const union mips_instruction breakpoint_insn = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	.b_format = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		.opcode = spec_op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 		.code = BRK_KPROBE_BP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 		.func = break_op
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) static const union mips_instruction breakpoint2_insn = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	.b_format = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 		.opcode = spec_op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		.code = BRK_KPROBE_SSTEPBP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 		.func = break_op
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) DEFINE_PER_CPU(struct kprobe *, current_kprobe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) static int __kprobes insn_has_delayslot(union mips_instruction insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	return __insn_has_delay_slot(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  * insn_has_ll_or_sc function checks whether instruction is ll or sc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52)  * one; putting breakpoint on top of atomic ll/sc pair is bad idea;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53)  * so we need to prevent it and refuse kprobes insertion for such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * instructions; cannot do much about breakpoint in the middle of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  * ll/sc pair; it is upto user to avoid those places
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) static int __kprobes insn_has_ll_or_sc(union mips_instruction insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	switch (insn.i_format.opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	case ll_op:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	case lld_op:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	case sc_op:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	case scd_op:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) int __kprobes arch_prepare_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	union mips_instruction insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	union mips_instruction prev_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	insn = p->addr[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	if (insn_has_ll_or_sc(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		pr_notice("Kprobes for ll and sc instructions are not"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 			  "supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	if (copy_from_kernel_nofault(&prev_insn, p->addr - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 			sizeof(mips_instruction)) == 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	    insn_has_delayslot(prev_insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		pr_notice("Kprobes for branch delayslot are not supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	if (__insn_is_compact_branch(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		pr_notice("Kprobes for compact branches are not supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	/* insn: must be on special executable page on mips. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	p->ainsn.insn = get_insn_slot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	if (!p->ainsn.insn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	 * In the kprobe->ainsn.insn[] array we store the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	 * instruction at index zero and a break trap instruction at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	 * index one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	 * On MIPS arch if the instruction at probed address is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	 * branch instruction, we need to execute the instruction at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	 * Branch Delayslot (BD) at the time of probe hit. As MIPS also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	 * doesn't have single stepping support, the BD instruction can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	 * not be executed in-line and it would be executed on SSOL slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	 * using a normal breakpoint instruction in the next slot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	 * So, read the instruction and save it for later execution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	if (insn_has_delayslot(insn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		memcpy(&p->ainsn.insn[0], p->addr + 1, sizeof(kprobe_opcode_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	p->ainsn.insn[1] = breakpoint2_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	p->opcode = *p->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) void __kprobes arch_arm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	*p->addr = breakpoint_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	flush_insn_slot(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) void __kprobes arch_disarm_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	*p->addr = p->opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	flush_insn_slot(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) void __kprobes arch_remove_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	if (p->ainsn.insn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		free_insn_slot(p->ainsn.insn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		p->ainsn.insn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static void save_previous_kprobe(struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	kcb->prev_kprobe.kp = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	kcb->prev_kprobe.status = kcb->kprobe_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	kcb->prev_kprobe.old_SR = kcb->kprobe_old_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	kcb->prev_kprobe.saved_SR = kcb->kprobe_saved_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	kcb->prev_kprobe.saved_epc = kcb->kprobe_saved_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	kcb->kprobe_status = kcb->prev_kprobe.status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	kcb->kprobe_old_SR = kcb->prev_kprobe.old_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	kcb->kprobe_saved_SR = kcb->prev_kprobe.saved_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	kcb->kprobe_saved_epc = kcb->prev_kprobe.saved_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 			       struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	__this_cpu_write(current_kprobe, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	kcb->kprobe_saved_SR = kcb->kprobe_old_SR = (regs->cp0_status & ST0_IE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	kcb->kprobe_saved_epc = regs->cp0_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)  * evaluate_branch_instrucion -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)  * Evaluate the branch instruction at probed address during probe hit. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)  * result of evaluation would be the updated epc. The insturction in delayslot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)  * would actually be single stepped using a normal breakpoint) on SSOL slot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)  * The result is also saved in the kprobe control block for later use,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)  * in case we need to execute the delayslot instruction. The latter will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)  * false for NOP instruction in dealyslot and the branch-likely instructions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)  * when the branch is taken. And for those cases we set a flag as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)  * SKIP_DELAYSLOT in the kprobe control block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 					struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	union mips_instruction insn = p->opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	long epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	epc = regs->cp0_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	if (epc & 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		goto unaligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	if (p->ainsn.insn->word == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 		kcb->flags |= SKIP_DELAYSLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		kcb->flags &= ~SKIP_DELAYSLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	ret = __compute_return_epc_for_insn(regs, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	if (ret == BRANCH_LIKELY_TAKEN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		kcb->flags |= SKIP_DELAYSLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	kcb->target_epc = regs->cp0_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unaligned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	pr_notice("%s: unaligned epc - sending SIGBUS.\n", current->comm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	force_sig(SIGBUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 						struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	regs->cp0_status &= ~ST0_IE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	/* single step inline if the instruction is a break */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	if (p->opcode.word == breakpoint_insn.word ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	    p->opcode.word == breakpoint2_insn.word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 		regs->cp0_epc = (unsigned long)p->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	else if (insn_has_delayslot(p->opcode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		ret = evaluate_branch_instruction(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 			pr_notice("Kprobes: Error in evaluating branch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	regs->cp0_epc = (unsigned long)&p->ainsn.insn[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  * Called after single-stepping.  p->addr is the address of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  * instruction whose first byte has been replaced by the "break 0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)  * instruction.	 To avoid the SMP problems that can occur when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)  * temporarily put back the original opcode to single-step, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)  * single-stepped a copy of the instruction.  The address of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)  * copy is p->ainsn.insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)  * This function prepares to return from the post-single-step
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)  * breakpoint trap. In case of branch instructions, the target
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)  * epc to be restored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static void __kprobes resume_execution(struct kprobe *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 				       struct pt_regs *regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 				       struct kprobe_ctlblk *kcb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	if (insn_has_delayslot(p->opcode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		regs->cp0_epc = kcb->target_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		unsigned long orig_epc = kcb->kprobe_saved_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		regs->cp0_epc = orig_epc + 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static int __kprobes kprobe_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	struct kprobe *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	kprobe_opcode_t *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	struct kprobe_ctlblk *kcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	addr = (kprobe_opcode_t *) regs->cp0_epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	 * We don't want to be preempted for the entire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	 * duration of kprobe processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	/* Check we're not actually recursing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	if (kprobe_running()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 		p = get_kprobe(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 			if (kcb->kprobe_status == KPROBE_HIT_SS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 			    p->ainsn.insn->word == breakpoint_insn.word) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 				regs->cp0_status &= ~ST0_IE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 				regs->cp0_status |= kcb->kprobe_saved_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 				goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 			 * We have reentered the kprobe_handler(), since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 			 * another probe was hit while within the handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 			 * We here save the original kprobes variables and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 			 * just single step on the instruction of the new probe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 			 * without calling any user handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 			save_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 			set_current_kprobe(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 			kprobes_inc_nmissed_count(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 			prepare_singlestep(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 			kcb->kprobe_status = KPROBE_REENTER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 			if (kcb->flags & SKIP_DELAYSLOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 				resume_execution(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 				restore_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 				preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 		} else if (addr->word != breakpoint_insn.word) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 			 * The breakpoint instruction was removed by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 			 * another cpu right after we hit, no further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 			 * handling of this interrupt is appropriate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 			ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 		goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	p = get_kprobe(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	if (!p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 		if (addr->word != breakpoint_insn.word) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 			 * The breakpoint instruction was removed right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 			 * after we hit it.  Another cpu has removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 			 * either a probepoint or a debugger breakpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 			 * at this address.  In either case, no further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 			 * handling of this interrupt is appropriate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 			ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 		/* Not one of ours: let kernel handle it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 		goto no_kprobe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	set_current_kprobe(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	if (p->pre_handler && p->pre_handler(p, regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		/* handler has already set things up, so skip ss setup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 		preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	prepare_singlestep(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	if (kcb->flags & SKIP_DELAYSLOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 		if (p->post_handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 			p->post_handler(p, regs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 		resume_execution(p, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 		preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	} else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 		kcb->kprobe_status = KPROBE_HIT_SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) no_kprobe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static inline int post_kprobe_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	struct kprobe *cur = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	if (!cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 		cur->post_handler(cur, regs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	resume_execution(cur, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	regs->cp0_status |= kcb->kprobe_saved_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	/* Restore back the original saved kprobes variables and continue. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	if (kcb->kprobe_status == KPROBE_REENTER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 		restore_previous_kprobe(kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	struct kprobe *cur = kprobe_running();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 	if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	if (kcb->kprobe_status & KPROBE_HIT_SS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 		resume_execution(cur, regs, kcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 		regs->cp0_status |= kcb->kprobe_old_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 		reset_current_kprobe();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 		preempt_enable_no_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)  * Wrapper routine for handling exceptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 				       unsigned long val, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 	struct die_args *args = (struct die_args *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	int ret = NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 	switch (val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	case DIE_BREAK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 		if (kprobe_handler(args->regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 			ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 	case DIE_SSTEPBP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 		if (post_kprobe_handler(args->regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 			ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	case DIE_PAGE_FAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 		/* kprobe_running() needs smp_processor_id() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 		preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 		if (kprobe_running()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 		    && kprobe_fault_handler(args->regs, args->trapnr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 			ret = NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 		preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)  * Function return probe trampoline:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)  *	- init_kprobes() establishes a probepoint here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)  *	- When the probed function returns, this probe causes the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)  *	  handlers to fire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static void __used kretprobe_trampoline_holder(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 		".set push\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 		/* Keep the assembler from reordering and placing JR here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 		".set noreorder\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 		"nop\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 		".global kretprobe_trampoline\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 		"kretprobe_trampoline:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 		"nop\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 		".set pop"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 		: : : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) void kretprobe_trampoline(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 				      struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 	ri->ret_addr = (kprobe_opcode_t *) regs->regs[31];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 	ri->fp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	/* Replace the return addr with trampoline addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 	regs->regs[31] = (unsigned long)kretprobe_trampoline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)  * Called when the probe at kretprobe trampoline is hit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static int __kprobes trampoline_probe_handler(struct kprobe *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 						struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 	instruction_pointer(regs) = __kretprobe_trampoline_handler(regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 						kretprobe_trampoline, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	 * By returning a non-zero value, we are telling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 	 * kprobe_handler() that we don't want the post_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	 * to run (and have re-enabled preemption)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) int __kprobes arch_trampoline_kprobe(struct kprobe *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	if (p->addr == (kprobe_opcode_t *)kretprobe_trampoline)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static struct kprobe trampoline_p = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 	.addr = (kprobe_opcode_t *)kretprobe_trampoline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	.pre_handler = trampoline_probe_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) int __init arch_init_kprobes(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 	return register_kprobe(&trampoline_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }