^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include "asm/cacheflush.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define JUMPLABEL_ERR "ARC: jump_label: ERROR: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) /* Halt system on fatal error to make debug easier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define arc_jl_fatal(format...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) pr_err(JUMPLABEL_ERR format); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) BUG(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) static inline u32 arc_gen_nop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /* 1x 32bit NOP in middle endian */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) return 0x7000264a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Atomic update of patched instruction is only available if this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * instruction doesn't cross L1 cache line boundary. You can read about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * the way we achieve this in arc/include/asm/jump_label.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static inline void instruction_align_assert(void *addr, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) unsigned long a = (unsigned long)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) if ((a >> L1_CACHE_SHIFT) != ((a + len - 1) >> L1_CACHE_SHIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) arc_jl_fatal("instruction (addr %px) cross L1 cache line border",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * ARCv2 'Branch unconditionally' instruction:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * 00000ssssssssss1SSSSSSSSSSNRtttt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * s S[n:0] lower bits signed immediate (number is bitfield size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * S S[m:n+1] upper bits signed immediate (number is bitfield size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * t S[24:21] upper bits signed immediate (branch unconditionally far)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * N N <.d> delay slot mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * R R Reserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static inline u32 arc_gen_branch(jump_label_t pc, jump_label_t target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) u32 instruction_l, instruction_r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u32 pcl = pc & GENMASK(31, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) u32 u_offset = target - pcl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) u32 s, S, t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * Offset in 32-bit branch instruction must to fit into s25.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * Something is terribly broken if we get such huge offset within one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if ((s32)u_offset < -16777216 || (s32)u_offset > 16777214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) arc_jl_fatal("gen branch with offset (%d) not fit in s25",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) (s32)u_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * All instructions are aligned by 2 bytes so we should never get offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * here which is not 2 bytes aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (u_offset & 0x1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) arc_jl_fatal("gen branch with offset (%d) unaligned to 2 bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) (s32)u_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) s = (u_offset >> 1) & GENMASK(9, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) S = (u_offset >> 11) & GENMASK(9, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) t = (u_offset >> 21) & GENMASK(3, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /* 00000ssssssssss1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) instruction_l = (s << 1) | 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* SSSSSSSSSSNRtttt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) instruction_r = (S << 6) | t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return (instruction_r << 16) | (instruction_l & GENMASK(15, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) void arch_jump_label_transform(struct jump_entry *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) enum jump_label_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) jump_label_t *instr_addr = (jump_label_t *)entry->code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u32 instr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) instruction_align_assert(instr_addr, JUMP_LABEL_NOP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (type == JUMP_LABEL_JMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) instr = arc_gen_branch(entry->code, entry->target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) instr = arc_gen_nop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) WRITE_ONCE(*instr_addr, instr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) flush_icache_range(entry->code, entry->code + JUMP_LABEL_NOP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) void arch_jump_label_transform_static(struct jump_entry *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) enum jump_label_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * We use only one NOP type (1x, 4 byte) in arch_static_branch, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * there's no need to patch an identical NOP over the top of it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * The generic code calls 'arch_jump_label_transform' if the NOP needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * to be replaced by a branch, so 'arch_jump_label_transform_static' is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * never called with type other than JUMP_LABEL_NOP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) BUG_ON(type != JUMP_LABEL_NOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #ifdef CONFIG_ARC_DBG_JUMP_LABEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define SELFTEST_MSG "ARC: instruction generation self-test: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct arc_gen_branch_testdata {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) jump_label_t pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) jump_label_t target_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u32 expected_instr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static __init int branch_gen_test(const struct arc_gen_branch_testdata *test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) u32 instr_got;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) instr_got = arc_gen_branch(test->pc, test->target_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (instr_got == test->expected_instr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) pr_err(SELFTEST_MSG "FAIL:\n arc_gen_branch(0x%08x, 0x%08x) != 0x%08x, got 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) test->pc, test->target_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) test->expected_instr, instr_got);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * Offset field in branch instruction is not continuous. Test all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * available offset field and sign combinations. Test data is generated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * from real working code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static const struct arc_gen_branch_testdata arcgenbr_test_data[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {0x90007548, 0x90007514, 0xffcf07cd}, /* tiny (-52) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {0x9000c9c0, 0x9000c782, 0xffcf05c3}, /* tiny (-574) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {0x9000cc1c, 0x9000c782, 0xffcf0367}, /* tiny (-1178) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {0x9009dce0, 0x9009d106, 0xff8f0427}, /* small (-3034) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {0x9000f5de, 0x90007d30, 0xfc0f0755}, /* big (-30892) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {0x900a2444, 0x90035f64, 0xc9cf0321}, /* huge (-443616) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {0x90007514, 0x9000752c, 0x00000019}, /* tiny (+24) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {0x9001a578, 0x9001a77a, 0x00000203}, /* tiny (+514) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {0x90031ed8, 0x90032634, 0x0000075d}, /* tiny (+1884) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {0x9008c7f2, 0x9008d3f0, 0x00400401}, /* small (+3072) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {0x9000bb38, 0x9003b340, 0x17c00009}, /* big (+194568) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {0x90008f44, 0x90578d80, 0xb7c2063d} /* huge (+5701180) offs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static __init int instr_gen_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) for (i = 0; i < ARRAY_SIZE(arcgenbr_test_data); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (branch_gen_test(&arcgenbr_test_data[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) pr_info(SELFTEST_MSG "OK\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) early_initcall(instr_gen_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #endif /* CONFIG_ARC_DBG_JUMP_LABEL */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags