^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/static_call.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <asm/text-patching.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) enum insn_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) CALL = 0, /* site call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) NOP = 1, /* site cond-call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) JMP = 2, /* tramp / site tail-call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) RET = 3, /* tramp / site cond-tail-call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) int size = CALL_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) const void *code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) case CALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) case NOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) code = ideal_nops[NOP_ATOMIC5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) case JMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) code = text_gen_insn(JMP32_INSN_OPCODE, insn, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) case RET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) code = text_gen_insn(RET_INSN_OPCODE, insn, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) size = RET_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) if (memcmp(insn, code, size) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (unlikely(system_state == SYSTEM_BOOTING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) return text_poke_early(insn, code, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) text_poke_bp(insn, code, size, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static void __static_call_validate(void *insn, bool tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u8 opcode = *(u8 *)insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (opcode == JMP32_INSN_OPCODE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) opcode == RET_INSN_OPCODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (opcode == CALL_INSN_OPCODE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) !memcmp(insn, ideal_nops[NOP_ATOMIC5], 5))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * If we ever trigger this, our text is corrupt, we'll probably not live long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static inline enum insn_type __sc_insn(bool null, bool tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Encode the following table without branches:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * tail null insn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * -----+-------+------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * 0 | 0 | CALL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * 0 | 1 | NOP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * 1 | 0 | JMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * 1 | 1 | RET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return 2*tail + null;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) mutex_lock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (tramp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) __static_call_validate(tramp, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) __static_call_transform(tramp, __sc_insn(!func, true), func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) __static_call_validate(site, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __static_call_transform(site, __sc_insn(!func, tail), func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) mutex_unlock(&text_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) EXPORT_SYMBOL_GPL(arch_static_call_transform);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags