^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2020 SiFive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/patch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct patch_insn {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) atomic_t cpu_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static void *patch_map(void *addr, int fixmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) uintptr_t uintaddr = (uintptr_t) addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) if (core_kernel_text(uintaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) page = phys_to_page(__pa_symbol(addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) page = vmalloc_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) BUG_ON(!page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) (uintaddr & ~PAGE_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) NOKPROBE_SYMBOL(patch_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static void patch_unmap(int fixmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) clear_fixmap(fixmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) NOKPROBE_SYMBOL(patch_unmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static int patch_insn_write(void *addr, const void *insn, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) void *waddr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * Before reaching here, it was expected to lock the text_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * already, so we don't need to give another lock here and could
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * ensure that it was safe between each cores.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) lockdep_assert_held(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if (across_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) patch_map(addr + len, FIX_TEXT_POKE1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) waddr = patch_map(addr, FIX_TEXT_POKE0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) ret = copy_to_kernel_nofault(waddr, insn, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) patch_unmap(FIX_TEXT_POKE0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (across_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) patch_unmap(FIX_TEXT_POKE1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) NOKPROBE_SYMBOL(patch_insn_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static int patch_insn_write(void *addr, const void *insn, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return copy_to_kernel_nofault(addr, insn, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) NOKPROBE_SYMBOL(patch_insn_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #endif /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int patch_text_nosync(void *addr, const void *insns, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u32 *tp = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) ret = patch_insn_write(tp, insns, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) NOKPROBE_SYMBOL(patch_text_nosync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static int patch_text_cb(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct patch_insn *patch = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (atomic_inc_return(&patch->cpu_count) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ret =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) patch_text_nosync(patch->addr, &patch->insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) GET_INSN_LENGTH(patch->insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) atomic_inc(&patch->cpu_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) while (atomic_read(&patch->cpu_count) <= num_online_cpus())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) NOKPROBE_SYMBOL(patch_text_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) int patch_text(void *addr, u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct patch_insn patch = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) .addr = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) .insn = insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) .cpu_count = ATOMIC_INIT(0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return stop_machine_cpuslocked(patch_text_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) &patch, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) NOKPROBE_SYMBOL(patch_text);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags