^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2014 ARM Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/sysreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/system_misc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include "trace-events-emulation.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * The runtime support for deprecated instruction support can be in one of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * following three states -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * 0 = undef
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * 1 = emulate (software emulation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * 2 = hw (supported in hardware)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) enum insn_emulation_mode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) INSN_UNDEF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) INSN_EMULATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) INSN_HW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) enum legacy_insn_status {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) INSN_DEPRECATED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) INSN_OBSOLETE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct insn_emulation_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) enum legacy_insn_status status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct undef_hook *hooks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int (*set_hw_mode)(bool enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct insn_emulation {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct insn_emulation_ops *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) int current_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) int min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) int max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static LIST_HEAD(insn_emulation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static int nr_insn_emulated __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static void register_emulation_hooks(struct insn_emulation_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct undef_hook *hook;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) BUG_ON(!ops->hooks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) for (hook = ops->hooks; hook->instr_mask; hook++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) register_undef_hook(hook);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) pr_notice("Registered %s emulation handler\n", ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static void remove_emulation_hooks(struct insn_emulation_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct undef_hook *hook;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) BUG_ON(!ops->hooks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) for (hook = ops->hooks; hook->instr_mask; hook++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unregister_undef_hook(hook);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) pr_notice("Removed %s emulation handler\n", ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static void enable_insn_hw_mode(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct insn_emulation *insn = (struct insn_emulation *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (insn->ops->set_hw_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) insn->ops->set_hw_mode(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static void disable_insn_hw_mode(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct insn_emulation *insn = (struct insn_emulation *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (insn->ops->set_hw_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) insn->ops->set_hw_mode(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /* Run set_hw_mode(mode) on all active CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (!insn->ops->set_hw_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * Run set_hw_mode for all insns on a starting CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * 0 - If all the hooks ran successfully.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * -EINVAL - At least one hook is not supported by the CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static int run_all_insn_set_hw_mode(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct insn_emulation *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) raw_spin_lock_irqsave(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) list_for_each_entry(insn, &insn_emulation, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) bool enable = (insn->current_mode == INSN_HW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) pr_warn("CPU[%u] cannot support the emulation of %s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) cpu, insn->ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static int update_insn_emulation_mode(struct insn_emulation *insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) enum insn_emulation_mode prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) switch (prev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) case INSN_UNDEF: /* Nothing to be done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) case INSN_EMULATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) remove_emulation_hooks(insn->ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) case INSN_HW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (!run_all_cpu_set_hw_mode(insn, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) pr_notice("Disabled %s support\n", insn->ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) switch (insn->current_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) case INSN_UNDEF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) case INSN_EMULATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) register_emulation_hooks(insn->ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) case INSN_HW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) ret = run_all_cpu_set_hw_mode(insn, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) pr_notice("Enabled %s support\n", insn->ops->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static void __init register_insn_emulation(struct insn_emulation_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct insn_emulation *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) insn = kzalloc(sizeof(*insn), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) insn->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) insn->min = INSN_UNDEF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) switch (ops->status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) case INSN_DEPRECATED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) insn->current_mode = INSN_EMULATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* Disable the HW mode if it was turned on at early boot time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) run_all_cpu_set_hw_mode(insn, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) insn->max = INSN_HW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) case INSN_OBSOLETE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) insn->current_mode = INSN_UNDEF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) insn->max = INSN_EMULATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) raw_spin_lock_irqsave(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) list_add(&insn->node, &insn_emulation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) nr_insn_emulated++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Register any handlers if required */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) update_insn_emulation_mode(insn, INSN_UNDEF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) static int emulation_proc_handler(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) void *buffer, size_t *lenp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct insn_emulation *insn = (struct insn_emulation *) table->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) enum insn_emulation_mode prev_mode = insn->current_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) table->data = &insn->current_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (ret || !write || prev_mode == insn->current_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) goto ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) ret = update_insn_emulation_mode(insn, prev_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) /* Mode change failed, revert to previous mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) insn->current_mode = prev_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) update_insn_emulation_mode(insn, INSN_UNDEF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) ret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) table->data = insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void __init register_insn_emulation_sysctl(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct insn_emulation *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct ctl_table *insns_sysctl, *sysctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) insns_sysctl = kcalloc(nr_insn_emulated + 1, sizeof(*sysctl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (!insns_sysctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) raw_spin_lock_irqsave(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) list_for_each_entry(insn, &insn_emulation, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) sysctl = &insns_sysctl[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) sysctl->mode = 0644;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) sysctl->maxlen = sizeof(int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) sysctl->procname = insn->ops->name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) sysctl->data = insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) sysctl->extra1 = &insn->min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) sysctl->extra2 = &insn->max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) sysctl->proc_handler = emulation_proc_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) register_sysctl("abi", insns_sysctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * Implement emulation of the SWP/SWPB instructions using load-exclusive and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * store-exclusive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * Where: Rt = destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * Rt2 = source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * Rn = address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /* Arbitrary constant to ensure forward-progress of the LL/SC loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #define __SWP_LL_SC_LOOPS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) #define __user_swpX_asm(data, addr, res, temp, temp2, B) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) uaccess_enable_privileged(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) " mov %w3, %w7\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) "0: ldxr"B" %w2, [%4]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) "1: stxr"B" %w0, %w1, [%4]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) " cbz %w0, 2f\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) " sub %w3, %w3, #1\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) " cbnz %w3, 0b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) " mov %w0, %w5\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) " b 3f\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) "2:\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) " mov %w1, %w2\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) "3:\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) " .pushsection .fixup,\"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) " .align 2\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) "4: mov %w0, %w6\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) " b 3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) " .popsection" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) _ASM_EXTABLE(0b, 4b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) _ASM_EXTABLE(1b, 4b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) : "r" ((unsigned long)addr), "i" (-EAGAIN), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) "i" (-EFAULT), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) "i" (__SWP_LL_SC_LOOPS) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) uaccess_disable_privileged(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #define __user_swp_asm(data, addr, res, temp, temp2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) __user_swpX_asm(data, addr, res, temp, temp2, "")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #define __user_swpb_asm(data, addr, res, temp, temp2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) __user_swpX_asm(data, addr, res, temp, temp2, "b")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * Bit 22 of the instruction encoding distinguishes between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * the SWP and SWPB variants (bit set means SWPB).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #define TYPE_SWPB (1 << 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static int emulate_swpX(unsigned int address, unsigned int *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) unsigned int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) unsigned int res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if ((type != TYPE_SWPB) && (address & 0x3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) /* SWP to unaligned address not permitted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) pr_debug("SWP instruction on unaligned pointer!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) unsigned long temp, temp2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (type == TYPE_SWPB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) __user_swpb_asm(*data, address, res, temp, temp2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) __user_swp_asm(*data, address, res, temp, temp2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (likely(res != -EAGAIN) || signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #define ARM_OPCODE_CONDTEST_FAIL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define ARM_OPCODE_CONDTEST_PASS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #define ARM_OPCODE_CONDTEST_UNCOND 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #define ARM_OPCODE_CONDITION_UNCOND 0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) u32 cc_bits = opcode >> 28;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return ARM_OPCODE_CONDTEST_PASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return ARM_OPCODE_CONDTEST_FAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return ARM_OPCODE_CONDTEST_UNCOND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * swp_handler logs the id of calling process, dissects the instruction, sanity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * checks the memory location, calls emulate_swpX for the actual operation and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * deals with fixup/error handling before returning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static int swp_handler(struct pt_regs *regs, u32 instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) u32 destreg, data, type, address = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) const void __user *user_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) int rn, rt2, res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) type = instr & TYPE_SWPB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) switch (aarch32_check_condition(instr, regs->pstate)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) case ARM_OPCODE_CONDTEST_PASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) case ARM_OPCODE_CONDTEST_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* Condition failed - return to next instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) goto ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) case ARM_OPCODE_CONDTEST_UNCOND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /* If unconditional encoding - not a SWP, undef */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) address = (u32)regs->user_regs.regs[rn];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) data = (u32)regs->user_regs.regs[rt2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) rn, address, destreg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* Check access in reasonable access range for both SWP and SWPB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) user_ptr = (const void __user *)(unsigned long)(address & ~3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (!access_ok(user_ptr, 4)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) goto fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) res = emulate_swpX(address, &data, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (res == -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) goto fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) else if (res == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) regs->user_regs.regs[destreg] = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) ret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (type == TYPE_SWPB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) trace_instruction_emulation("swpb", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) trace_instruction_emulation("swp", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) current->comm, (unsigned long)current->pid, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) arm64_skip_faulting_instruction(regs, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) pr_debug("SWP{B} emulation: access caused memory abort!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) arm64_notify_segfault(address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * Only emulate SWP/SWPB executed in ARM state/User mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * The kernel must be SWP free and SWP{B} does not exist in Thumb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static struct undef_hook swp_hooks[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) .instr_mask = 0x0fb00ff0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) .instr_val = 0x01000090,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) .pstate_mask = PSR_AA32_MODE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) .pstate_val = PSR_AA32_MODE_USR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) .fn = swp_handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) static struct insn_emulation_ops swp_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) .name = "swp",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) .status = INSN_OBSOLETE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) .hooks = swp_hooks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) .set_hw_mode = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) switch (aarch32_check_condition(instr, regs->pstate)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) case ARM_OPCODE_CONDTEST_PASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) case ARM_OPCODE_CONDTEST_FAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /* Condition failed - return to next instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) goto ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) case ARM_OPCODE_CONDTEST_UNCOND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) /* If unconditional encoding - not a barrier instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) switch (aarch32_insn_mcr_extract_crm(instr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) case 10:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * dmb - mcr p15, 0, Rt, c7, c10, 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * dsb - mcr p15, 0, Rt, c7, c10, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) dmb(sy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) trace_instruction_emulation(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) dsb(sy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) trace_instruction_emulation(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) case 5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * isb - mcr p15, 0, Rt, c7, c5, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * Taking an exception or returning from one acts as an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * instruction barrier. So no explicit barrier needed here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) trace_instruction_emulation(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) current->comm, (unsigned long)current->pid, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) arm64_skip_faulting_instruction(regs, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static int cp15_barrier_set_hw_mode(bool enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) static struct undef_hook cp15_barrier_hooks[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) .instr_mask = 0x0fff0fdf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) .instr_val = 0x0e070f9a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) .pstate_mask = PSR_AA32_MODE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) .pstate_val = PSR_AA32_MODE_USR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) .fn = cp15barrier_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) .instr_mask = 0x0fff0fff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) .instr_val = 0x0e070f95,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) .pstate_mask = PSR_AA32_MODE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) .pstate_val = PSR_AA32_MODE_USR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) .fn = cp15barrier_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) static struct insn_emulation_ops cp15_barrier_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) .name = "cp15_barrier",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) .status = INSN_DEPRECATED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) .hooks = cp15_barrier_hooks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) .set_hw_mode = cp15_barrier_set_hw_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) static int setend_set_hw_mode(bool enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (!cpu_supports_mixed_endian_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) char *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (big_endian) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) insn = "setend be";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) regs->pstate |= PSR_AA32_E_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) insn = "setend le";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) regs->pstate &= ~PSR_AA32_E_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) trace_instruction_emulation(insn, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) current->comm, (unsigned long)current->pid, regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) static int a32_setend_handler(struct pt_regs *regs, u32 instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int rc = compat_setend_handler(regs, (instr >> 9) & 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) arm64_skip_faulting_instruction(regs, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) static int t16_setend_handler(struct pt_regs *regs, u32 instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) int rc = compat_setend_handler(regs, (instr >> 3) & 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) arm64_skip_faulting_instruction(regs, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) static struct undef_hook setend_hooks[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) .instr_mask = 0xfffffdff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) .instr_val = 0xf1010000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) .pstate_mask = PSR_AA32_MODE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) .pstate_val = PSR_AA32_MODE_USR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) .fn = a32_setend_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /* Thumb mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) .instr_mask = 0xfffffff7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) .instr_val = 0x0000b650,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) .pstate_mask = (PSR_AA32_T_BIT | PSR_AA32_MODE_MASK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) .pstate_val = (PSR_AA32_T_BIT | PSR_AA32_MODE_USR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) .fn = t16_setend_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static struct insn_emulation_ops setend_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) .name = "setend",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) .status = INSN_DEPRECATED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) .hooks = setend_hooks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) .set_hw_mode = setend_set_hw_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * Invoked as core_initcall, which guarantees that the instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * emulation is ready for userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) static int __init armv8_deprecated_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (IS_ENABLED(CONFIG_SWP_EMULATION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) register_insn_emulation(&swp_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) register_insn_emulation(&cp15_barrier_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (system_supports_mixed_endian_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) register_insn_emulation(&setend_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) pr_info("setend instruction emulation is not supported on this system\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) "arm64/isndep:starting",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) run_all_insn_set_hw_mode, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) register_insn_emulation_sysctl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) core_initcall(armv8_deprecated_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags