^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #define pr_fmt(fmt) "SMP alternatives: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/stringify.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/bsearch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/sync_core.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/text-patching.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/mce.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) int __read_mostly alternatives_patched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) EXPORT_SYMBOL_GPL(alternatives_patched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define MAX_PATCH_LEN (255-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static int __initdata_or_module debug_alternative;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int __init debug_alt(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) debug_alternative = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) __setup("debug-alternative", debug_alt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static int noreplace_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static int __init setup_noreplace_smp(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) noreplace_smp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) __setup("noreplace-smp", setup_noreplace_smp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define DPRINTK(fmt, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (debug_alternative) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define DUMP_BYTES(buf, len, fmt, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (unlikely(debug_alternative)) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) int j; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if (!(len)) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) printk(KERN_DEBUG pr_fmt(fmt), ##args); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) for (j = 0; j < (len) - 1; j++) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) printk(KERN_CONT "%02hhx ", buf[j]); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) printk(KERN_CONT "%02hhx\n", buf[j]); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * that correspond to that nop. Getting from one nop to the next, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * add to the array the offset that is equal to the sum of all sizes of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * nops preceding the one we are after.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * nice symmetry of sizes of the previous nops.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static const unsigned char intelnops[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) GENERIC_NOP1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) GENERIC_NOP2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) GENERIC_NOP3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) GENERIC_NOP4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) GENERIC_NOP5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) GENERIC_NOP6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) GENERIC_NOP7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) GENERIC_NOP8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) GENERIC_NOP5_ATOMIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) intelnops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) intelnops + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) intelnops + 1 + 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) intelnops + 1 + 2 + 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) intelnops + 1 + 2 + 3 + 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) intelnops + 1 + 2 + 3 + 4 + 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) intelnops + 1 + 2 + 3 + 4 + 5 + 6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #ifdef K8_NOP1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static const unsigned char k8nops[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) K8_NOP1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) K8_NOP2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) K8_NOP3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) K8_NOP4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) K8_NOP5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) K8_NOP6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) K8_NOP7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) K8_NOP8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) K8_NOP5_ATOMIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) k8nops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) k8nops + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) k8nops + 1 + 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) k8nops + 1 + 2 + 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) k8nops + 1 + 2 + 3 + 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) k8nops + 1 + 2 + 3 + 4 + 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) k8nops + 1 + 2 + 3 + 4 + 5 + 6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static const unsigned char k7nops[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) K7_NOP1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) K7_NOP2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) K7_NOP3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) K7_NOP4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) K7_NOP5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) K7_NOP6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) K7_NOP7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) K7_NOP8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) K7_NOP5_ATOMIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) k7nops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) k7nops + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) k7nops + 1 + 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) k7nops + 1 + 2 + 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) k7nops + 1 + 2 + 3 + 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) k7nops + 1 + 2 + 3 + 4 + 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) k7nops + 1 + 2 + 3 + 4 + 5 + 6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #ifdef P6_NOP1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static const unsigned char p6nops[] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) P6_NOP1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) P6_NOP2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) P6_NOP3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) P6_NOP4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) P6_NOP5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) P6_NOP6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) P6_NOP7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) P6_NOP8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) P6_NOP5_ATOMIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) p6nops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) p6nops + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) p6nops + 1 + 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) p6nops + 1 + 2 + 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) p6nops + 1 + 2 + 3 + 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) p6nops + 1 + 2 + 3 + 4 + 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) p6nops + 1 + 2 + 3 + 4 + 5 + 6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /* Initialize these to a safe default */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) const unsigned char * const *ideal_nops = p6_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) const unsigned char * const *ideal_nops = intel_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) void __init arch_init_ideal_nops(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) switch (boot_cpu_data.x86_vendor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) case X86_VENDOR_INTEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * Due to a decoder implementation quirk, some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * specific Intel CPUs actually perform better with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * the "k8_nops" than with the SDM-recommended NOPs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (boot_cpu_data.x86 == 6 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) boot_cpu_data.x86_model >= 0x0f &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) boot_cpu_data.x86_model != 0x1c &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) boot_cpu_data.x86_model != 0x26 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) boot_cpu_data.x86_model != 0x27 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) boot_cpu_data.x86_model < 0x30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) ideal_nops = k8_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) } else if (boot_cpu_has(X86_FEATURE_NOPL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) ideal_nops = p6_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) ideal_nops = k8_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) ideal_nops = intel_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) case X86_VENDOR_HYGON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) ideal_nops = p6_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) case X86_VENDOR_AMD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) if (boot_cpu_data.x86 > 0xf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) ideal_nops = p6_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ideal_nops = k8_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (boot_cpu_has(X86_FEATURE_K8))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ideal_nops = k8_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) else if (boot_cpu_has(X86_FEATURE_K7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) ideal_nops = k7_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) ideal_nops = intel_nops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /* Use this to add nops to a buffer, then text_poke the whole buffer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) static void __init_or_module add_nops(void *insns, unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) while (len > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) unsigned int noplen = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (noplen > ASM_NOP_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) noplen = ASM_NOP_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) memcpy(insns, ideal_nops[noplen], noplen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) insns += noplen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) len -= noplen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) extern s32 __smp_locks[], __smp_locks_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) void text_poke_early(void *addr, const void *opcode, size_t len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * Are we looking at a near JMP with a 1 or 4-byte displacement.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static inline bool is_jmp(const u8 opcode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return opcode == 0xeb || opcode == 0xe9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static void __init_or_module
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) u8 *next_rip, *tgt_rip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) s32 n_dspl, o_dspl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int repl_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (a->replacementlen != 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) o_dspl = *(s32 *)(insn_buff + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /* next_rip of the replacement JMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) next_rip = repl_insn + a->replacementlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* target rip of the replacement JMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) tgt_rip = next_rip + o_dspl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) n_dspl = tgt_rip - orig_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (tgt_rip - orig_insn >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (n_dspl - 2 <= 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) goto two_byte_jmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) goto five_byte_jmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* negative offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (((n_dspl - 2) & 0xff) == (n_dspl - 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto two_byte_jmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) goto five_byte_jmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) two_byte_jmp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) n_dspl -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) insn_buff[0] = 0xeb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) insn_buff[1] = (s8)n_dspl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) add_nops(insn_buff + 2, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) repl_len = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) five_byte_jmp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) n_dspl -= 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) insn_buff[0] = 0xe9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) *(s32 *)&insn_buff[1] = n_dspl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) repl_len = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) DPRINTK("final displ: 0x%08x, JMP 0x%lx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * "noinline" to cause control flow change and thus invalidate I$ and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * cause refetch after modification.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) for (i = 0; i < a->padlen; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (instr[i] != 0x90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) add_nops(instr + (a->instrlen - a->padlen), a->padlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) instr, a->instrlen - a->padlen, a->padlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * Replace instructions with better alternatives for this CPU type. This runs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * before SMP is initialized to avoid SMP problems with self modifying code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * This implies that asymmetric systems where APs have less capabilities than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * the boot processor are not handled. Tough. Make sure you disable such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * features by hand.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Marked "noinline" to cause control flow change and thus insn cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * to refetch changed I$ lines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) void __init_or_module noinline apply_alternatives(struct alt_instr *start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct alt_instr *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct alt_instr *a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) u8 *instr, *replacement;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) u8 insn_buff[MAX_PATCH_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) DPRINTK("alt table %px, -> %px", start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * The scan order should be from start to end. A later scanned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * alternative code can overwrite previously scanned alternative code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * Some kernel functions (e.g. memcpy, memset, etc) use this order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * patch code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * So be careful if you want to change the scan order to any other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) for (a = start; a < end; a++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) int insn_buff_sz = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) instr = (u8 *)&a->instr_offset + a->instr_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) replacement = (u8 *)&a->repl_offset + a->repl_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) BUG_ON(a->instrlen > sizeof(insn_buff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (!boot_cpu_has(a->cpuid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (a->padlen > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) optimize_nops(a, instr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) a->cpuid >> 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) a->cpuid & 0x1f,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) instr, instr, a->instrlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) replacement, a->replacementlen, a->padlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) memcpy(insn_buff, replacement, a->replacementlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) insn_buff_sz = a->replacementlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * 0xe8 is a relative jump; fix the offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * Instruction length is checked before the opcode to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * accessing uninitialized bytes for zero-length replacements.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (a->replacementlen == 5 && *insn_buff == 0xe8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) *(s32 *)(insn_buff + 1) += replacement - instr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) *(s32 *)(insn_buff + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) (unsigned long)instr + *(s32 *)(insn_buff + 1) + 5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (a->replacementlen && is_jmp(replacement[0]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) recompute_jump(a, instr, replacement, insn_buff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (a->instrlen > a->replacementlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) add_nops(insn_buff + a->replacementlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) a->instrlen - a->replacementlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) insn_buff_sz += a->instrlen - a->replacementlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) text_poke_early(instr, insn_buff, insn_buff_sz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static void alternatives_smp_lock(const s32 *start, const s32 *end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) u8 *text, u8 *text_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) const s32 *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) for (poff = start; poff < end; poff++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) u8 *ptr = (u8 *)poff + *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) if (!*poff || ptr < text || ptr >= text_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /* turn DS segment override prefix into lock prefix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (*ptr == 0x3e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) text_poke(ptr, ((unsigned char []){0xf0}), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) static void alternatives_smp_unlock(const s32 *start, const s32 *end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) u8 *text, u8 *text_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) const s32 *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) for (poff = start; poff < end; poff++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) u8 *ptr = (u8 *)poff + *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (!*poff || ptr < text || ptr >= text_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* turn lock prefix into DS segment override prefix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (*ptr == 0xf0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) text_poke(ptr, ((unsigned char []){0x3E}), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct smp_alt_module {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /* what is this ??? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct module *mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) /* ptrs to lock prefixes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) const s32 *locks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) const s32 *locks_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /* .text segment, needed to avoid patching init code ;) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) u8 *text;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) u8 *text_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct list_head next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) static LIST_HEAD(smp_alt_modules);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) static bool uniproc_patched = false; /* protected by text_mutex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) void __init_or_module alternatives_smp_module_add(struct module *mod,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) char *name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) void *locks, void *locks_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) void *text, void *text_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct smp_alt_module *smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) mutex_lock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (!uniproc_patched)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (num_possible_cpus() == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /* Don't bother remembering, we'll never have to undo it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) goto smp_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) smp = kzalloc(sizeof(*smp), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (NULL == smp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /* we'll run the (safe but slow) SMP code then ... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) smp->mod = mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) smp->name = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) smp->locks = locks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) smp->locks_end = locks_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) smp->text = text;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) smp->text_end = text_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) smp->locks, smp->locks_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) smp->text, smp->text_end, smp->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) list_add_tail(&smp->next, &smp_alt_modules);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) smp_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) alternatives_smp_unlock(locks, locks_end, text, text_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) mutex_unlock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) void __init_or_module alternatives_smp_module_del(struct module *mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct smp_alt_module *item;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) mutex_lock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) list_for_each_entry(item, &smp_alt_modules, next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (mod != item->mod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) list_del(&item->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) kfree(item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) mutex_unlock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) void alternatives_enable_smp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct smp_alt_module *mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /* Why bother if there are no other CPUs? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) BUG_ON(num_possible_cpus() == 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) mutex_lock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (uniproc_patched) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) pr_info("switching to SMP code\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) BUG_ON(num_online_cpus() != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) list_for_each_entry(mod, &smp_alt_modules, next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) alternatives_smp_lock(mod->locks, mod->locks_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) mod->text, mod->text_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) uniproc_patched = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) mutex_unlock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * Return 1 if the address range is reserved for SMP-alternatives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * Must hold text_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) int alternatives_text_reserved(void *start, void *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct smp_alt_module *mod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) const s32 *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) u8 *text_start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) u8 *text_end = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) lockdep_assert_held(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) list_for_each_entry(mod, &smp_alt_modules, next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (mod->text > text_end || mod->text_end < text_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) for (poff = mod->locks; poff < mod->locks_end; poff++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) const u8 *ptr = (const u8 *)poff + *poff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) if (text_start <= ptr && text_end > ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) #ifdef CONFIG_PARAVIRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) struct paravirt_patch_site *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) struct paravirt_patch_site *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) char insn_buff[MAX_PATCH_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) for (p = start; p < end; p++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) unsigned int used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) BUG_ON(p->len > MAX_PATCH_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) /* prep the buffer with the original instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) memcpy(insn_buff, p->instr, p->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) BUG_ON(used > p->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) /* Pad the rest with nops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) add_nops(insn_buff + used, p->len - used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) text_poke_early(p->instr, insn_buff, p->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) extern struct paravirt_patch_site __start_parainstructions[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) __stop_parainstructions[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) #endif /* CONFIG_PARAVIRT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * Self-test for the INT3 based CALL emulation code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * properly and that there is a stack gap between the INT3 frame and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * previous context. Without this gap doing a virtual PUSH on the interrupted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * stack would corrupt the INT3 IRET frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * See entry_{32,64}.S for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static void __init __no_sanitize_address notrace int3_magic(unsigned int *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) *ptr = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) extern __initdata unsigned long int3_selftest_ip; /* defined in asm below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) static int __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct die_args *args = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) struct pt_regs *regs = args->regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (!regs || user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (val != DIE_INT3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) int3_emulate_call(regs, (unsigned long)&int3_magic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return NOTIFY_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) static void __init int3_selftest(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static __initdata struct notifier_block int3_exception_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) .notifier_call = int3_exception_notify,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) .priority = INT_MAX-1, /* last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) unsigned int val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) BUG_ON(register_die_notifier(&int3_exception_nb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * Basically: int3_magic(&val); but really complicated :-)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * Stick the address of the INT3 instruction into int3_selftest_ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * then trigger the INT3, padded with NOPs to match a CALL instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) asm volatile ("1: int3; nop; nop; nop; nop\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ".pushsection .init.data,\"aw\"\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ".align " __ASM_SEL(4, 8) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ".type int3_selftest_ip, @object\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) ".size int3_selftest_ip, " __ASM_SEL(4, 8) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) "int3_selftest_ip:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) __ASM_SEL(.long, .quad) " 1b\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) : ASM_CALL_CONSTRAINT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) : __ASM_SEL_RAW(a, D) (&val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) BUG_ON(val != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) unregister_die_notifier(&int3_exception_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) void __init alternative_instructions(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) int3_selftest();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * The patching is not fully atomic, so try to avoid local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * interruptions that might execute the to be patched code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * Other CPUs are not running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) stop_nmi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * Don't stop machine check exceptions while patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * MCEs only happen when something got corrupted and in this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * case we must do something about the corruption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * Ignoring it is worse than an unlikely patching race.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * Also machine checks tend to be broadcast and if one CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * goes into machine check the others follow quickly, so we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * expect a machine check to cause undue problems during to code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) apply_alternatives(__alt_instructions, __alt_instructions_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) /* Patch to UP if other cpus not imminent. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) uniproc_patched = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) alternatives_smp_module_add(NULL, "core kernel",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) __smp_locks, __smp_locks_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) _text, _etext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (!uniproc_patched || num_possible_cpus() == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) free_init_pages("SMP alternatives",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) (unsigned long)__smp_locks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) (unsigned long)__smp_locks_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) apply_paravirt(__parainstructions, __parainstructions_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) restart_nmi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) alternatives_patched = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) * text_poke_early - Update instructions on a live kernel at boot time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * @addr: address to modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * @opcode: source of the copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * @len: length to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * When you use this code to patch more than one byte of an instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * you need to make sure that other CPUs cannot execute this code in parallel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * Also no thread must be currently preempted in the middle of these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * instructions. And on the local CPU you need to be protected against NMI or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * MCE handlers seeing an inconsistent instruction while you patch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) void __init_or_module text_poke_early(void *addr, const void *opcode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) if (boot_cpu_has(X86_FEATURE_NX) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) is_module_text_address((unsigned long)addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * Modules text is marked initially as non-executable, so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * code cannot be running and speculative code-fetches are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * prevented. Just change the code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) memcpy(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) memcpy(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) sync_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * Could also do a CLFLUSH here to speed up CPU recovery; but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * that causes hangs on some VIA CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) typedef struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) } temp_mm_state_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * Using a temporary mm allows to set temporary mappings that are not accessible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * by other CPUs. Such mappings are needed to perform sensitive memory writes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * that override the kernel memory protections (e.g., W^X), without exposing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * temporary page-table mappings that are required for these write operations to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * mapping is torn down.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * Context: The temporary mm needs to be used exclusively by a single core. To
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * harden security IRQs must be disabled while the temporary mm is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * loaded, thereby preventing interrupt handler bugs from overriding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * the kernel memory protection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) temp_mm_state_t temp_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * Make sure not to be in TLB lazy mode, as otherwise we'll end up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * with a stale address space WITHOUT being in lazy mode after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * restoring the previous mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (this_cpu_read(cpu_tlbstate.is_lazy))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) leave_mm(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) switch_mm_irqs_off(NULL, mm, current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * If breakpoints are enabled, disable them while the temporary mm is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * used. Userspace might set up watchpoints on addresses that are used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * in the temporary mm, which would lead to wrong signals being sent or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * crashes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * Note that breakpoints are not disabled selectively, which also causes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * kernel breakpoints (e.g., perf's) to be disabled. This might be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * undesirable, but still seems reasonable as the code that runs in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * temporary mm should be short.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (hw_breakpoint_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) hw_breakpoint_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return temp_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) switch_mm_irqs_off(NULL, prev_state.mm, current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * Restore the breakpoints if they were disabled before the temporary mm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * was loaded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (hw_breakpoint_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) hw_breakpoint_restore();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) __ro_after_init struct mm_struct *poking_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) __ro_after_init unsigned long poking_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) static void *__text_poke(void *addr, const void *opcode, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct page *pages[2] = {NULL};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) temp_mm_state_t prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) pte_t pte, *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) pgprot_t pgprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * While boot memory allocator is running we cannot use struct pages as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * they are not yet initialized. There is no way to recover.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) BUG_ON(!after_bootmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (!core_kernel_text((unsigned long)addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) pages[0] = vmalloc_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) if (cross_page_boundary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) pages[0] = virt_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) WARN_ON(!PageReserved(pages[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) if (cross_page_boundary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) pages[1] = virt_to_page(addr + PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * If something went wrong, crash and burn since recovery paths are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * implemented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * Map the page without the global bit, as TLB flushing is done with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * flush_tlb_mm_range(), which is intended for non-global PTEs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * The lock is not really needed, but this allows to avoid open-coding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) ptep = get_locked_pte(poking_mm, poking_addr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * This must not fail; preallocated in poking_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) VM_BUG_ON(!ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) pte = mk_pte(pages[0], pgprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) set_pte_at(poking_mm, poking_addr, ptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) if (cross_page_boundary) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) pte = mk_pte(pages[1], pgprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) set_pte_at(poking_mm, poking_addr + PAGE_SIZE, ptep + 1, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * Loading the temporary mm behaves as a compiler barrier, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * guarantees that the PTE will be set at the time memcpy() is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) prev = use_temporary_mm(poking_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) kasan_disable_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) memcpy((u8 *)poking_addr + offset_in_page(addr), opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) kasan_enable_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * Ensure that the PTE is only cleared after the instructions of memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * were issued by using a compiler barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) pte_clear(poking_mm, poking_addr, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (cross_page_boundary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) pte_clear(poking_mm, poking_addr + PAGE_SIZE, ptep + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * Loading the previous page-table hierarchy requires a serializing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * instruction that already allows the core to see the updated version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * Xen-PV is assumed to serialize execution in a similar manner.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) unuse_temporary_mm(prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * Flushing the TLB might involve IPIs, which would require enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * IRQs, but not if the mm is not used, as it is in this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) flush_tlb_mm_range(poking_mm, poking_addr, poking_addr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) PAGE_SHIFT, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * If the text does not match what we just wrote then something is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * fundamentally screwy; there's nothing we can really do about that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) BUG_ON(memcmp(addr, opcode, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) pte_unmap_unlock(ptep, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * text_poke - Update instructions on a live kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * @addr: address to modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * @opcode: source of the copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * @len: length to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Only atomic text poke/set should be allowed when not doing early patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * It means the size must be writable atomically and the address must be aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * in a way that permits an atomic write. It also makes sure we fit on a single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * Note that the caller must ensure that if the modified code is part of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * module, the module would not be removed during poking. This can be achieved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * by registering a module notifier, and ordering module removal and patching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * trough a mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) void *text_poke(void *addr, const void *opcode, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) lockdep_assert_held(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) return __text_poke(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * text_poke_kgdb - Update instructions on a live kernel by kgdb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * @addr: address to modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) * @opcode: source of the copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) * @len: length to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * Only atomic text poke/set should be allowed when not doing early patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * It means the size must be writable atomically and the address must be aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * in a way that permits an atomic write. It also makes sure we fit on a single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * Context: should only be used by kgdb, which ensures no other core is running,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * despite the fact it does not hold the text_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) return __text_poke(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) static void do_sync_core(void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) sync_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) void text_poke_sync(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) on_each_cpu(do_sync_core, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) struct text_poke_loc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) s32 rel_addr; /* addr := _stext + rel_addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) s32 rel32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) u8 opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) const u8 text[POKE_MAX_OPCODE_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) u8 old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct bp_patching_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) struct text_poke_loc *vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) atomic_t refs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) static struct bp_patching_desc *bp_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) static __always_inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) if (!desc || !arch_atomic_inc_not_zero(&desc->refs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) return desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) static __always_inline void put_desc(struct bp_patching_desc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) arch_atomic_dec(&desc->refs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) static __always_inline void *text_poke_addr(struct text_poke_loc *tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return _stext + tp->rel_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) static __always_inline int patch_cmp(const void *key, const void *elt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) struct text_poke_loc *tp = (struct text_poke_loc *) elt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) if (key < text_poke_addr(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (key > text_poke_addr(tp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) noinstr int poke_int3_handler(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) struct bp_patching_desc *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) struct text_poke_loc *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) int len, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) void *ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * Having observed our INT3 instruction, we now must observe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * bp_desc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) * bp_desc = desc INT3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) * WMB RMB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * write INT3 if (desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) desc = try_get_desc(&bp_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) if (!desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * Discount the INT3. See text_poke_bp_batch().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) ip = (void *) regs->ip - INT3_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * Skip the binary search if there is a single member in the vector.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (unlikely(desc->nr_entries > 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) sizeof(struct text_poke_loc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) patch_cmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (!tp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) tp = desc->vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (text_poke_addr(tp) != ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) len = text_opcode_size(tp->opcode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) ip += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) switch (tp->opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) case INT3_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * Someone poked an explicit INT3, they'll want to handle it,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * do not consume.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) case RET_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) int3_emulate_ret(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) case CALL_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) int3_emulate_call(regs, (long)ip + tp->rel32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) case JMP32_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) case JMP8_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) int3_emulate_jmp(regs, (long)ip + tp->rel32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) put_desc(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) #define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static struct text_poke_loc tp_vec[TP_VEC_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) static int tp_vec_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * text_poke_bp_batch() -- update instructions on live kernel on SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) * @tp: vector of instructions to patch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) * @nr_entries: number of entries in the vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * Modify multi-byte instruction by using int3 breakpoint on SMP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * We completely avoid stop_machine() here, and achieve the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * synchronization using int3 breakpoint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) * The way it is done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * - For each entry in the vector:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * - add a int3 trap to the address that will be patched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * - sync cores
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) * - For each entry in the vector:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * - update all but the first byte of the patched range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * - sync cores
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * - For each entry in the vector:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * - replace the first byte (int3) by the first byte of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * replacing opcode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * - sync cores
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) struct bp_patching_desc desc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) .vec = tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) .nr_entries = nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) .refs = ATOMIC_INIT(1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) unsigned char int3 = INT3_INSN_OPCODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) int do_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) lockdep_assert_held(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) smp_store_release(&bp_desc, &desc); /* rcu_assign_pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * Corresponding read barrier in int3 notifier for making sure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * nr_entries and handler are correctly ordered wrt. patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) * First step: add a int3 trap to the address that will be patched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) for (i = 0; i < nr_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) tp[i].old = *(u8 *)text_poke_addr(&tp[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) text_poke(text_poke_addr(&tp[i]), &int3, INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) text_poke_sync();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * Second step: update all but the first byte of the patched range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) for (do_sync = 0, i = 0; i < nr_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) u8 old[POKE_MAX_OPCODE_SIZE] = { tp[i].old, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) int len = text_opcode_size(tp[i].opcode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (len - INT3_INSN_SIZE > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) memcpy(old + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) len - INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) text_poke(text_poke_addr(&tp[i]) + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) (const char *)tp[i].text + INT3_INSN_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) len - INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) do_sync++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * Emit a perf event to record the text poke, primarily to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * support Intel PT decoding which must walk the executable code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) * to reconstruct the trace. The flow up to here is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * - write INT3 byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * - IPI-SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * - write instruction tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * At this point the actual control flow will be through the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) * INT3 and handler and not hit the old or new instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * Intel PT outputs FUP/TIP packets for the INT3, so the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * can still be decoded. Subsequently:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * - emit RECORD_TEXT_POKE with the new instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * - IPI-SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * - write first byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) * - IPI-SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) * So before the text poke event timestamp, the decoder will see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * either the old instruction flow or FUP/TIP of INT3. After the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * text poke event timestamp, the decoder will see either the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * new instruction flow or FUP/TIP of INT3. Thus decoders can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * use the timestamp as the point at which to modify the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * executable code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * The old instruction is recorded so that the event can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * processed forwards or backwards.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) perf_event_text_poke(text_poke_addr(&tp[i]), old, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) tp[i].text, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (do_sync) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) * According to Intel, this core syncing is very likely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) * not necessary and we'd be safe even without it. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * better safe than sorry (plus there's not only Intel).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) text_poke_sync();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * Third step: replace the first byte (int3) by the first byte of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * replacing opcode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) for (do_sync = 0, i = 0; i < nr_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (tp[i].text[0] == INT3_INSN_OPCODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) text_poke(text_poke_addr(&tp[i]), tp[i].text, INT3_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) do_sync++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) if (do_sync)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) text_poke_sync();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * Remove and synchronize_rcu(), except we have a very primitive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * refcount based completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) WRITE_ONCE(bp_desc, NULL); /* RCU_INIT_POINTER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (!atomic_dec_and_test(&desc.refs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) atomic_cond_read_acquire(&desc.refs, !VAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) const void *opcode, size_t len, const void *emulate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct insn insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) memcpy((void *)tp->text, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) if (!emulate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) emulate = opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) kernel_insn_init(&insn, emulate, MAX_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) insn_get_length(&insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) BUG_ON(!insn_complete(&insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) BUG_ON(len != insn.length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) tp->rel_addr = addr - (void *)_stext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) tp->opcode = insn.opcode.bytes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) switch (tp->opcode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) case INT3_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) case RET_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) case CALL_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) case JMP32_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) case JMP8_INSN_OPCODE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) tp->rel32 = insn.immediate.value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) default: /* assume NOP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) switch (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) case 2: /* NOP2 -- emulate as JMP8+0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) BUG_ON(memcmp(emulate, ideal_nops[len], len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) tp->opcode = JMP8_INSN_OPCODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) tp->rel32 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) case 5: /* NOP5 -- emulate as JMP32+0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) BUG_ON(memcmp(emulate, ideal_nops[NOP_ATOMIC5], len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) tp->opcode = JMP32_INSN_OPCODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) tp->rel32 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) default: /* unknown instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * We hard rely on the tp_vec being ordered; ensure this is so by flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) * early if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) static bool tp_order_fail(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) struct text_poke_loc *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) if (!tp_vec_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (!addr) /* force */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) tp = &tp_vec[tp_vec_nr - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) if ((unsigned long)text_poke_addr(tp) > (unsigned long)addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) static void text_poke_flush(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (tp_vec_nr == TP_VEC_MAX || tp_order_fail(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) text_poke_bp_batch(tp_vec, tp_vec_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) tp_vec_nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) void text_poke_finish(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) text_poke_flush(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const void *emulate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) struct text_poke_loc *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) if (unlikely(system_state == SYSTEM_BOOTING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) text_poke_early(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) text_poke_flush(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) tp = &tp_vec[tp_vec_nr++];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) text_poke_loc_init(tp, addr, opcode, len, emulate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) * text_poke_bp() -- update instructions on live kernel on SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) * @addr: address to patch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * @opcode: opcode of new instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * @len: length to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * @handler: address to jump to when the temporary breakpoint is hit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * Update a single instruction with the vector in the stack, avoiding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * dynamically allocated memory. This function should be used when it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * not possible to allocate memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void *emulate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) struct text_poke_loc tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) if (unlikely(system_state == SYSTEM_BOOTING)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) text_poke_early(addr, opcode, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) text_poke_loc_init(&tp, addr, opcode, len, emulate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) text_poke_bp_batch(&tp, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags