^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * using the CPU's debug registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2012 ARM Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Will Deacon <will.deacon@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define pr_fmt(fmt) "hw-breakpoint: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cpu_pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/hw_breakpoint.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/current.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/debug-monitors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/hw_breakpoint.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/cputype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/system_misc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* Breakpoint currently in use for each BRP. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /* Watchpoint currently in use for each WRP. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* Currently stepping a per-CPU kernel breakpoint. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static DEFINE_PER_CPU(int, stepping_kernel_bp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /* Number of BRP/WRP registers on this CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static int core_num_brps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int core_num_wrps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) int hw_breakpoint_slots(int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * We can be called early, so don't rely on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * our static variables being initialised.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) case TYPE_INST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return get_num_brps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) case TYPE_DATA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return get_num_wrps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) pr_warn("unknown slot type: %d\n", type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define READ_WB_REG_CASE(OFF, N, REG, VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) case (OFF + N): \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) AARCH64_DBG_READ(N, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) break
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) case (OFF + N): \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) AARCH64_DBG_WRITE(N, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) break
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) READ_WB_REG_CASE(OFF, 0, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) READ_WB_REG_CASE(OFF, 1, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) READ_WB_REG_CASE(OFF, 2, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) READ_WB_REG_CASE(OFF, 3, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) READ_WB_REG_CASE(OFF, 4, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) READ_WB_REG_CASE(OFF, 5, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) READ_WB_REG_CASE(OFF, 6, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) READ_WB_REG_CASE(OFF, 7, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) READ_WB_REG_CASE(OFF, 8, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) READ_WB_REG_CASE(OFF, 9, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) READ_WB_REG_CASE(OFF, 10, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) READ_WB_REG_CASE(OFF, 11, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) READ_WB_REG_CASE(OFF, 12, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) READ_WB_REG_CASE(OFF, 13, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) READ_WB_REG_CASE(OFF, 14, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) READ_WB_REG_CASE(OFF, 15, REG, VAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static u64 read_wb_reg(int reg, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u64 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) switch (reg + n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pr_warn("attempt to read from unknown breakpoint register %d\n", n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) NOKPROBE_SYMBOL(read_wb_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static void write_wb_reg(int reg, int n, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) switch (reg + n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) pr_warn("attempt to write to unknown breakpoint register %d\n", n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) NOKPROBE_SYMBOL(write_wb_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * Convert a breakpoint privilege level to the corresponding exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static enum dbg_active_el debug_exception_level(int privilege)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) switch (privilege) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) case AARCH64_BREAKPOINT_EL0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return DBG_ACTIVE_EL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) case AARCH64_BREAKPOINT_EL1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return DBG_ACTIVE_EL1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) pr_warn("invalid breakpoint privilege level %d\n", privilege);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) NOKPROBE_SYMBOL(debug_exception_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) enum hw_breakpoint_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) HW_BREAKPOINT_INSTALL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) HW_BREAKPOINT_UNINSTALL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) HW_BREAKPOINT_RESTORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static int is_compat_bp(struct perf_event *bp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct task_struct *tsk = bp->hw.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * In this case, use the native interface, since we don't have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * the notion of a "compat CPU" and could end up relying on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * deprecated behaviour if we use unaligned watchpoints in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * AArch64 state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return tsk && is_compat_thread(task_thread_info(tsk));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * hw_breakpoint_slot_setup - Find and setup a perf slot according to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * @slots: pointer to array of slots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * @max_slots: max number of slots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * @bp: perf_event to setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * @ops: operation to be carried out on the slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * slot index on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * -ENOSPC if no slot is available/matches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * -EINVAL on wrong operations parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct perf_event *bp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) enum hw_breakpoint_ops ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct perf_event **slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) for (i = 0; i < max_slots; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) slot = &slots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) switch (ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) case HW_BREAKPOINT_INSTALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (!*slot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) *slot = bp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) case HW_BREAKPOINT_UNINSTALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (*slot == bp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) *slot = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) case HW_BREAKPOINT_RESTORE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (*slot == bp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static int hw_breakpoint_control(struct perf_event *bp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) enum hw_breakpoint_ops ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct arch_hw_breakpoint *info = counter_arch_bp(bp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) struct perf_event **slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) struct debug_info *debug_info = ¤t->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) int i, max_slots, ctrl_reg, val_reg, reg_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) u32 ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* Breakpoint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) ctrl_reg = AARCH64_DBG_REG_BCR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) val_reg = AARCH64_DBG_REG_BVR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) slots = this_cpu_ptr(bp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) max_slots = core_num_brps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) reg_enable = !debug_info->bps_disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /* Watchpoint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) ctrl_reg = AARCH64_DBG_REG_WCR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) val_reg = AARCH64_DBG_REG_WVR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) slots = this_cpu_ptr(wp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) max_slots = core_num_wrps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) reg_enable = !debug_info->wps_disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) switch (ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) case HW_BREAKPOINT_INSTALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * Ensure debug monitors are enabled at the correct exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) enable_debug_monitors(dbg_el);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) case HW_BREAKPOINT_RESTORE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* Setup the address register. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) write_wb_reg(val_reg, i, info->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /* Setup the control register. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) ctrl = encode_ctrl_reg(info->ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) write_wb_reg(ctrl_reg, i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) case HW_BREAKPOINT_UNINSTALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /* Reset the control register. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) write_wb_reg(ctrl_reg, i, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * Release the debug monitors for the correct exception
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) disable_debug_monitors(dbg_el);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * Install a perf counter breakpoint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int arch_install_hw_breakpoint(struct perf_event *bp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) void arch_uninstall_hw_breakpoint(struct perf_event *bp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static int get_hbp_len(u8 hbp_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) unsigned int len_in_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) switch (hbp_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) case ARM_BREAKPOINT_LEN_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) len_in_bytes = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) case ARM_BREAKPOINT_LEN_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) len_in_bytes = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) case ARM_BREAKPOINT_LEN_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) len_in_bytes = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) case ARM_BREAKPOINT_LEN_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) len_in_bytes = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) case ARM_BREAKPOINT_LEN_5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) len_in_bytes = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) case ARM_BREAKPOINT_LEN_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) len_in_bytes = 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) case ARM_BREAKPOINT_LEN_7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) len_in_bytes = 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) case ARM_BREAKPOINT_LEN_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) len_in_bytes = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return len_in_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * Check whether bp virtual address is in kernel space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) unsigned long va;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) va = hw->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) len = get_hbp_len(hw->ctrl.len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * Hopefully this will disappear when ptrace can bypass the conversion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * to generic breakpoint descriptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) int *gen_len, int *gen_type, int *offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* Type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) switch (ctrl.type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) case ARM_BREAKPOINT_EXECUTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) *gen_type = HW_BREAKPOINT_X;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) case ARM_BREAKPOINT_LOAD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) *gen_type = HW_BREAKPOINT_R;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) case ARM_BREAKPOINT_STORE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) *gen_type = HW_BREAKPOINT_W;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) *gen_type = HW_BREAKPOINT_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (!ctrl.len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *offset = __ffs(ctrl.len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /* Len */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) switch (ctrl.len >> *offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) case ARM_BREAKPOINT_LEN_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) *gen_len = HW_BREAKPOINT_LEN_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) case ARM_BREAKPOINT_LEN_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) *gen_len = HW_BREAKPOINT_LEN_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) case ARM_BREAKPOINT_LEN_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) *gen_len = HW_BREAKPOINT_LEN_3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) case ARM_BREAKPOINT_LEN_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) *gen_len = HW_BREAKPOINT_LEN_4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) case ARM_BREAKPOINT_LEN_5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) *gen_len = HW_BREAKPOINT_LEN_5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) case ARM_BREAKPOINT_LEN_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) *gen_len = HW_BREAKPOINT_LEN_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) case ARM_BREAKPOINT_LEN_7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) *gen_len = HW_BREAKPOINT_LEN_7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) case ARM_BREAKPOINT_LEN_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) *gen_len = HW_BREAKPOINT_LEN_8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * Construct an arch_hw_breakpoint from a perf_event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static int arch_build_bp_info(struct perf_event *bp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) const struct perf_event_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct arch_hw_breakpoint *hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /* Type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) switch (attr->bp_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) case HW_BREAKPOINT_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) case HW_BREAKPOINT_R:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) hw->ctrl.type = ARM_BREAKPOINT_LOAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) case HW_BREAKPOINT_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) hw->ctrl.type = ARM_BREAKPOINT_STORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) case HW_BREAKPOINT_RW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /* Len */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) switch (attr->bp_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) case HW_BREAKPOINT_LEN_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) case HW_BREAKPOINT_LEN_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) case HW_BREAKPOINT_LEN_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) hw->ctrl.len = ARM_BREAKPOINT_LEN_3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) case HW_BREAKPOINT_LEN_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) case HW_BREAKPOINT_LEN_5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) hw->ctrl.len = ARM_BREAKPOINT_LEN_5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) case HW_BREAKPOINT_LEN_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) hw->ctrl.len = ARM_BREAKPOINT_LEN_6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) case HW_BREAKPOINT_LEN_7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) hw->ctrl.len = ARM_BREAKPOINT_LEN_7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) case HW_BREAKPOINT_LEN_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * On AArch64, we only permit breakpoints of length 4, whereas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * AArch32 also requires breakpoints of length 2 for Thumb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Watchpoints can be of length 1, 2, 4 or 8 bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (is_compat_bp(bp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * FIXME: Some tools (I'm looking at you perf) assume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * that breakpoints should be sizeof(long). This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * is nonsense. For now, we fix up the parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * but we should probably return -EINVAL instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /* Address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) hw->address = attr->bp_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * Privilege
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * Note that we disallow combined EL0/EL1 breakpoints because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * that would complicate the stepping code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (arch_check_bp_in_kernelspace(hw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /* Enabled? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) hw->ctrl.enabled = !attr->disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * Validate the arch-specific HW Breakpoint register settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) int hw_breakpoint_arch_parse(struct perf_event *bp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) const struct perf_event_attr *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct arch_hw_breakpoint *hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) u64 alignment_mask, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) /* Build the arch_hw_breakpoint. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ret = arch_build_bp_info(bp, attr, hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * Check address alignment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * We don't do any clever alignment correction for watchpoints
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * because using 64-bit unaligned addresses is deprecated for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * AArch64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * AArch32 tasks expect some simple alignment fixups, so emulate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * that here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (is_compat_bp(bp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) alignment_mask = 0x7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) alignment_mask = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) offset = hw->address & alignment_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) switch (offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /* Aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /* Allow halfword watchpoints and breakpoints. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) case 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /* Allow single byte watchpoint. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) alignment_mask = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) alignment_mask = 0x7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) offset = hw->address & alignment_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) hw->address &= ~alignment_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) hw->ctrl.len <<= offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * Disallow per-task kernel breakpoints since these would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * complicate the stepping code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * Enable/disable all of the breakpoints active at the specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * exception level at the register level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * This is used when single-stepping after a breakpoint exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int i, max_slots, privilege;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) u32 ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) struct perf_event **slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) switch (reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) case AARCH64_DBG_REG_BCR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) slots = this_cpu_ptr(bp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) max_slots = core_num_brps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) case AARCH64_DBG_REG_WCR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) slots = this_cpu_ptr(wp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) max_slots = core_num_wrps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) for (i = 0; i < max_slots; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (!slots[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) privilege = counter_arch_bp(slots[i])->ctrl.privilege;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (debug_exception_level(privilege) != el)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) ctrl = read_wb_reg(reg, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) ctrl |= 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) ctrl &= ~0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) write_wb_reg(reg, i, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) NOKPROBE_SYMBOL(toggle_bp_registers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * Debug exception handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static int breakpoint_handler(unsigned long unused, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) int i, step = 0, *kernel_step;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) u32 ctrl_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) u64 addr, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) struct perf_event *bp, **slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct debug_info *debug_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct arch_hw_breakpoint_ctrl ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) slots = this_cpu_ptr(bp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) addr = instruction_pointer(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) debug_info = ¤t->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) for (i = 0; i < core_num_brps; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) bp = slots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (bp == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /* Check if the breakpoint value matches. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (val != (addr & ~0x3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) /* Possible match, check the byte address select to confirm. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) decode_ctrl_reg(ctrl_reg, &ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (!((1 << (addr & 0x3)) & ctrl.len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) counter_arch_bp(bp)->trigger = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) perf_bp_event(bp, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* Do we need to handle the stepping? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (is_default_overflow_handler(bp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) step = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (!step)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) debug_info->bps_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /* If we're already stepping a watchpoint, just return. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (debug_info->wps_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (test_thread_flag(TIF_SINGLESTEP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) debug_info->suspended_step = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) user_enable_single_step(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) kernel_step = this_cpu_ptr(&stepping_kernel_bp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (*kernel_step != ARM_KERNEL_STEP_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (kernel_active_single_step()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) *kernel_step = ARM_KERNEL_STEP_SUSPEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) *kernel_step = ARM_KERNEL_STEP_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) kernel_enable_single_step(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) NOKPROBE_SYMBOL(breakpoint_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * Arm64 hardware does not always report a watchpoint hit address that matches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * one of the watchpoints set. It can also report an address "near" the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * watchpoint if a single instruction access both watched and unwatched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * addresses. There is no straight-forward way, short of disassembling the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * offending instruction, to map that address back to the watchpoint. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * function computes the distance of the memory access from the watchpoint as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * heuristic for the likelyhood that a given access triggered the watchpoint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * exception" of ARMv8 Architecture Reference Manual for details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * The function returns the distance of the address from the bytes watched by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * the watchpoint. In case of an exact match, it returns 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) struct arch_hw_breakpoint_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) u64 wp_low, wp_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) u32 lens, lene;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) addr = untagged_addr(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) lens = __ffs(ctrl->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) lene = __fls(ctrl->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) wp_low = val + lens;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) wp_high = val + lene;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (addr < wp_low)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) return wp_low - addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) else if (addr > wp_high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return addr - wp_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static int watchpoint_report(struct perf_event *wp, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) int step = is_default_overflow_handler(wp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) struct arch_hw_breakpoint *info = counter_arch_bp(wp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) info->trigger = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * If we triggered a user watchpoint from a uaccess routine, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * handle the stepping ourselves since userspace really can't help
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * us with this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (!user_mode(regs) && info->ctrl.privilege == AARCH64_BREAKPOINT_EL0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) step = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) perf_bp_event(wp, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return step;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) static int watchpoint_handler(unsigned long addr, unsigned int esr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) int i, step = 0, *kernel_step, access, closest_match = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) u64 min_dist = -1, dist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) u32 ctrl_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) struct perf_event *wp, **slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) struct debug_info *debug_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) struct arch_hw_breakpoint_ctrl ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) slots = this_cpu_ptr(wp_on_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) debug_info = ¤t->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * Find all watchpoints that match the reported address. If no exact
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * match is found. Attribute the hit to the closest watchpoint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) for (i = 0; i < core_num_wrps; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) wp = slots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (wp == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * Check that the access type matches.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * 0 => load, otherwise => store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) HW_BREAKPOINT_R;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) if (!(access & hw_breakpoint_type(wp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /* Check if the watchpoint value and byte select match. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) decode_ctrl_reg(ctrl_reg, &ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) dist = get_distance_from_watchpoint(addr, val, &ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (dist < min_dist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) min_dist = dist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) closest_match = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /* Is this an exact match? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (dist != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) step = watchpoint_report(wp, addr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) /* No exact match found? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (min_dist > 0 && min_dist != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) step = watchpoint_report(slots[closest_match], addr, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) if (!step)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * We always disable EL0 watchpoints because the kernel can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * cause these to fire via an unprivileged access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) debug_info->wps_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /* If we're already stepping a breakpoint, just return. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) if (debug_info->bps_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (test_thread_flag(TIF_SINGLESTEP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) debug_info->suspended_step = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) user_enable_single_step(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) kernel_step = this_cpu_ptr(&stepping_kernel_bp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (*kernel_step != ARM_KERNEL_STEP_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (kernel_active_single_step()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) *kernel_step = ARM_KERNEL_STEP_SUSPEND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) *kernel_step = ARM_KERNEL_STEP_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) kernel_enable_single_step(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) NOKPROBE_SYMBOL(watchpoint_handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * Handle single-step exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) int reinstall_suspended_bps(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct debug_info *debug_info = ¤t->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) int handled_exception = 0, *kernel_step;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) kernel_step = this_cpu_ptr(&stepping_kernel_bp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * Called from single-step exception handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * Return 0 if execution can resume, 1 if a SIGTRAP should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * reported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (debug_info->bps_disabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) debug_info->bps_disabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) handled_exception = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (debug_info->wps_disabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) debug_info->wps_disabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) handled_exception = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (handled_exception) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (debug_info->suspended_step) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) debug_info->suspended_step = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /* Allow exception handling to fall-through. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) handled_exception = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) user_disable_single_step(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (!debug_info->wps_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) kernel_disable_single_step();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) handled_exception = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) handled_exception = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) *kernel_step = ARM_KERNEL_STEP_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) return !handled_exception;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) NOKPROBE_SYMBOL(reinstall_suspended_bps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) * Context-switcher for restoring suspended breakpoints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) void hw_breakpoint_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * current next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * disabled: 0 0 => The usual case, NOTIFY_DONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * 0 1 => Disable the registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * 1 0 => Enable the registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * 1 1 => NOTIFY_DONE. per-task bps will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * get taken care of by perf.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) struct debug_info *current_debug_info, *next_debug_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) current_debug_info = ¤t->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) next_debug_info = &next->thread.debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /* Update breakpoints. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) toggle_bp_registers(AARCH64_DBG_REG_BCR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) DBG_ACTIVE_EL0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) !next_debug_info->bps_disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /* Update watchpoints. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) toggle_bp_registers(AARCH64_DBG_REG_WCR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) DBG_ACTIVE_EL0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) !next_debug_info->wps_disabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * CPU initialisation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) static int hw_breakpoint_reset(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) struct perf_event **slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) * When a CPU goes through cold-boot, it does not have any installed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * slot, so it is safe to share the same function for restoring and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * resetting breakpoints; when a CPU is hotplugged in, it goes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * through the slots, which are all empty, hence it just resets control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * and value for debug registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * When this function is triggered on warm-boot through a CPU PM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * notifier some slots might be initialized; if so they are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * reprogrammed according to the debug slots content.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (slots[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (slots[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) #ifdef CONFIG_CPU_PM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) extern void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) static inline void cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * One-time initialisation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) static int __init arch_hw_breakpoint_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) core_num_brps = get_num_brps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) core_num_wrps = get_num_wrps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) pr_info("found %d breakpoint and %d watchpoint registers.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) core_num_brps, core_num_wrps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) /* Register debug fault handlers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) TRAP_HWBKPT, "hw-breakpoint handler");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) TRAP_HWBKPT, "hw-watchpoint handler");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * Reset the breakpoint resources. We assume that a halting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * debugger will leave the world in a nice state for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) ret = cpuhp_setup_state(CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) "perf/arm64/hw_breakpoint:starting",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) hw_breakpoint_reset, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) pr_err("failed to register CPU hotplug notifier: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) /* Register cpu_suspend hw breakpoint restore hook */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) arch_initcall(arch_hw_breakpoint_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) void hw_breakpoint_pmu_read(struct perf_event *bp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * Dummy function to register with die_notifier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) unsigned long val, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags