^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2013 Huawei Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author: Jiang Liu <liuj97@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/debug-monitors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/kprobes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define AARCH64_INSN_SF_BIT BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define AARCH64_INSN_N_BIT BIT(22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define AARCH64_INSN_LSL_12 BIT(22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static const int aarch64_insn_encoding_class[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) AARCH64_INSN_CLS_UNKNOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) AARCH64_INSN_CLS_UNKNOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) AARCH64_INSN_CLS_UNKNOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) AARCH64_INSN_CLS_UNKNOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) AARCH64_INSN_CLS_LDST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) AARCH64_INSN_CLS_DP_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) AARCH64_INSN_CLS_LDST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) AARCH64_INSN_CLS_DP_FPSIMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) AARCH64_INSN_CLS_DP_IMM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) AARCH64_INSN_CLS_DP_IMM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) AARCH64_INSN_CLS_BR_SYS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) AARCH64_INSN_CLS_BR_SYS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) AARCH64_INSN_CLS_LDST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) AARCH64_INSN_CLS_DP_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) AARCH64_INSN_CLS_LDST,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) AARCH64_INSN_CLS_DP_FPSIMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (!aarch64_insn_is_hint(insn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) switch (insn & 0xFE0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) case AARCH64_INSN_HINT_XPACLRI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) case AARCH64_INSN_HINT_PACIA_1716:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) case AARCH64_INSN_HINT_PACIB_1716:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) case AARCH64_INSN_HINT_PACIAZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) case AARCH64_INSN_HINT_PACIASP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) case AARCH64_INSN_HINT_PACIBZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) case AARCH64_INSN_HINT_PACIBSP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) case AARCH64_INSN_HINT_BTI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) case AARCH64_INSN_HINT_BTIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) case AARCH64_INSN_HINT_BTIJ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) case AARCH64_INSN_HINT_BTIJC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) case AARCH64_INSN_HINT_NOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) bool aarch64_insn_is_branch_imm(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) aarch64_insn_is_bcond(insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static DEFINE_RAW_SPINLOCK(patch_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static bool is_exit_text(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* discarded with init text/data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return system_state < SYSTEM_RUNNING &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) addr >= (unsigned long)__exittext_begin &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) addr < (unsigned long)__exittext_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static bool is_image_text(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return core_kernel_text(addr) || is_exit_text(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static void __kprobes *patch_map(void *addr, int fixmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) unsigned long uintaddr = (uintptr_t) addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) bool image = is_image_text(uintaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (image)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) page = phys_to_page(__pa_symbol(addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) page = vmalloc_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) BUG_ON(!page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) (uintaddr & ~PAGE_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static void __kprobes patch_unmap(int fixmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) clear_fixmap(fixmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * little-endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) __le32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) *insnp = le32_to_cpu(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) void *waddr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) unsigned long flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) raw_spin_lock_irqsave(&patch_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) waddr = patch_map(addr, FIX_TEXT_POKE0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) patch_unmap(FIX_TEXT_POKE0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) raw_spin_unlock_irqrestore(&patch_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) int __kprobes aarch64_insn_write(void *addr, u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return __aarch64_insn_write(addr, cpu_to_le32(insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) bool __kprobes aarch64_insn_uses_literal(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* ldr/ldrsw (literal), prfm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return aarch64_insn_is_ldr_lit(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) aarch64_insn_is_ldrsw_lit(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) aarch64_insn_is_adr_adrp(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) aarch64_insn_is_prfm_lit(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) bool __kprobes aarch64_insn_is_branch(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return aarch64_insn_is_b(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) aarch64_insn_is_bl(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) aarch64_insn_is_cbz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) aarch64_insn_is_cbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) aarch64_insn_is_tbz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) aarch64_insn_is_tbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) aarch64_insn_is_ret(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) aarch64_insn_is_ret_auth(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) aarch64_insn_is_br(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) aarch64_insn_is_br_auth(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) aarch64_insn_is_blr(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) aarch64_insn_is_blr_auth(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) aarch64_insn_is_bcond(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) u32 *tp = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* A64 instructions must be word aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if ((uintptr_t)tp & 0x3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) ret = aarch64_insn_write(tp, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) __flush_icache_range((uintptr_t)tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) (uintptr_t)tp + AARCH64_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct aarch64_insn_patch {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) void **text_addrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) u32 *new_insns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int insn_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) atomic_t cpu_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static int __kprobes aarch64_insn_patch_text_cb(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) int i, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct aarch64_insn_patch *pp = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /* The first CPU becomes master */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (atomic_inc_return(&pp->cpu_count) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) pp->new_insns[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) /* Notify other processors with an additional increment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) atomic_inc(&pp->cpu_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) while (atomic_read(&pp->cpu_count) <= num_online_cpus())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct aarch64_insn_patch patch = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) .text_addrs = addrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) .new_insns = insns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) .insn_cnt = cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) .cpu_count = ATOMIC_INIT(0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (cnt <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) u32 *maskp, int *shiftp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) case AARCH64_INSN_IMM_26:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) mask = BIT(26) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) case AARCH64_INSN_IMM_19:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) mask = BIT(19) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) shift = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) case AARCH64_INSN_IMM_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) mask = BIT(16) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) shift = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) case AARCH64_INSN_IMM_14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) mask = BIT(14) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) shift = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) case AARCH64_INSN_IMM_12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) mask = BIT(12) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) shift = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) case AARCH64_INSN_IMM_9:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) mask = BIT(9) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) shift = 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) case AARCH64_INSN_IMM_7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) mask = BIT(7) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) shift = 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) case AARCH64_INSN_IMM_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) case AARCH64_INSN_IMM_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) mask = BIT(6) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) shift = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) case AARCH64_INSN_IMM_R:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) mask = BIT(6) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) shift = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) case AARCH64_INSN_IMM_N:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) mask = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) shift = 22;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) *maskp = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) *shiftp = shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) #define ADR_IMM_HILOSPLIT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #define ADR_IMM_SIZE SZ_2M
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #define ADR_IMM_LOSHIFT 29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #define ADR_IMM_HISHIFT 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) u32 immlo, immhi, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) case AARCH64_INSN_IMM_ADR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) mask = ADR_IMM_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return (insn >> shift) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) u32 insn, u64 imm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) u32 immlo, immhi, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (insn == AARCH64_BREAK_FAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) case AARCH64_INSN_IMM_ADR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) imm >>= ADR_IMM_HILOSPLIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) imm = immlo | immhi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* Update the immediate field. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) insn &= ~(mask << shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) insn |= (imm & mask) << shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) case AARCH64_INSN_REGTYPE_RT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) case AARCH64_INSN_REGTYPE_RD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) case AARCH64_INSN_REGTYPE_RN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) shift = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) case AARCH64_INSN_REGTYPE_RT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) case AARCH64_INSN_REGTYPE_RA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) shift = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) case AARCH64_INSN_REGTYPE_RM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) shift = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) pr_err("%s: unknown register type encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return (insn >> shift) & GENMASK(4, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) u32 insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) enum aarch64_insn_register reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (insn == AARCH64_BREAK_FAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) pr_err("%s: unknown register encoding %d\n", __func__, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) case AARCH64_INSN_REGTYPE_RT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) case AARCH64_INSN_REGTYPE_RD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) case AARCH64_INSN_REGTYPE_RN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) shift = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) case AARCH64_INSN_REGTYPE_RT2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) case AARCH64_INSN_REGTYPE_RA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) shift = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) case AARCH64_INSN_REGTYPE_RM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) case AARCH64_INSN_REGTYPE_RS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) shift = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) pr_err("%s: unknown register type encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) insn &= ~(GENMASK(4, 0) << shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) insn |= reg << shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) case AARCH64_INSN_SIZE_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) case AARCH64_INSN_SIZE_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) size = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) case AARCH64_INSN_SIZE_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) size = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) case AARCH64_INSN_SIZE_64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) size = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) pr_err("%s: unknown size encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) insn &= ~GENMASK(31, 30);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) insn |= size << 30;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) static inline long branch_imm_common(unsigned long pc, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) long range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if ((pc & 0x3) || (addr & 0x3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) pr_err("%s: A64 instructions must be word aligned\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) offset = ((long)addr - (long)pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (offset < -range || offset >= range) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) pr_err("%s: offset out of range\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) enum aarch64_insn_branch_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * B/BL support [-128M, 128M) offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * ARM64 virtual address arrangement guarantees all kernel and module
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * texts are within +/-128M.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) offset = branch_imm_common(pc, addr, SZ_128M);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (offset >= SZ_128M)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) case AARCH64_INSN_BRANCH_LINK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) insn = aarch64_insn_get_bl_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) case AARCH64_INSN_BRANCH_NOLINK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) insn = aarch64_insn_get_b_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) pr_err("%s: unknown branch encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) enum aarch64_insn_branch_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) offset = branch_imm_common(pc, addr, SZ_1M);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (offset >= SZ_1M)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) case AARCH64_INSN_BRANCH_COMP_ZERO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) insn = aarch64_insn_get_cbz_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) case AARCH64_INSN_BRANCH_COMP_NONZERO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) insn = aarch64_insn_get_cbnz_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) pr_err("%s: unknown branch encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) enum aarch64_insn_condition cond)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) offset = branch_imm_common(pc, addr, SZ_1M);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) insn = aarch64_insn_get_bcond_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) pr_err("%s: unknown condition encoding %d\n", __func__, cond);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) insn |= cond;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return aarch64_insn_get_hint_value() | op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) u32 __kprobes aarch64_insn_gen_nop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) enum aarch64_insn_branch_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) case AARCH64_INSN_BRANCH_NOLINK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) insn = aarch64_insn_get_br_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) case AARCH64_INSN_BRANCH_LINK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) insn = aarch64_insn_get_blr_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) case AARCH64_INSN_BRANCH_RETURN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) insn = aarch64_insn_get_ret_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) pr_err("%s: unknown branch encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) enum aarch64_insn_register base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) enum aarch64_insn_register offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) enum aarch64_insn_size_type size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) enum aarch64_insn_ldst_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) insn = aarch64_insn_get_ldr_reg_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) case AARCH64_INSN_LDST_STORE_REG_OFFSET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) insn = aarch64_insn_get_str_reg_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) pr_err("%s: unknown load/store encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) insn = aarch64_insn_encode_ldst_size(size, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) enum aarch64_insn_register reg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) enum aarch64_insn_register base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) enum aarch64_insn_ldst_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) insn = aarch64_insn_get_ldp_pre_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) insn = aarch64_insn_get_stp_pre_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) insn = aarch64_insn_get_ldp_post_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) insn = aarch64_insn_get_stp_post_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) pr_err("%s: unknown load/store encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) __func__, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) shift = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) __func__, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) shift = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) reg1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) reg2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) offset >> shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) enum aarch64_insn_register base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) enum aarch64_insn_register state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) enum aarch64_insn_size_type size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) enum aarch64_insn_ldst_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) case AARCH64_INSN_LDST_LOAD_EX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) insn = aarch64_insn_get_load_ex_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) case AARCH64_INSN_LDST_STORE_EX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) insn = aarch64_insn_get_store_ex_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) insn = aarch64_insn_encode_ldst_size(size, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) AARCH64_INSN_REG_ZR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) enum aarch64_insn_register address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) enum aarch64_insn_register value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) enum aarch64_insn_size_type size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) u32 insn = aarch64_insn_get_ldadd_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) switch (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) case AARCH64_INSN_SIZE_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) case AARCH64_INSN_SIZE_64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) pr_err("%s: unimplemented size encoding %d\n", __func__, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) insn = aarch64_insn_encode_ldst_size(size, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) enum aarch64_insn_register value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) enum aarch64_insn_size_type size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * STADD is simply encoded as an alias for LDADD with XZR as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * the destination register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return aarch64_insn_gen_ldadd(AARCH64_INSN_REG_ZR, address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) value, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) enum aarch64_insn_prfm_target target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) enum aarch64_insn_prfm_policy policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) u32 imm_type = 0, imm_target = 0, imm_policy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) case AARCH64_INSN_PRFM_TYPE_PLD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) case AARCH64_INSN_PRFM_TYPE_PLI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) imm_type = BIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) case AARCH64_INSN_PRFM_TYPE_PST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) imm_type = BIT(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) pr_err("%s: unknown prfm type encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) switch (target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) case AARCH64_INSN_PRFM_TARGET_L1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) case AARCH64_INSN_PRFM_TARGET_L2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) imm_target = BIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) case AARCH64_INSN_PRFM_TARGET_L3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) imm_target = BIT(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) pr_err("%s: unknown prfm target encoding %d\n", __func__, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) switch (policy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) case AARCH64_INSN_PRFM_POLICY_KEEP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) case AARCH64_INSN_PRFM_POLICY_STRM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) imm_policy = BIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /* In this case, imm5 is encoded into Rt field. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) insn &= ~GENMASK(4, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) insn |= imm_policy | (imm_target << 1) | (imm_type << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) return insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) enum aarch64_insn_prfm_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) enum aarch64_insn_prfm_target target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) enum aarch64_insn_prfm_policy policy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) u32 insn = aarch64_insn_get_prfm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) int imm, enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) enum aarch64_insn_adsb_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) case AARCH64_INSN_ADSB_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) insn = aarch64_insn_get_add_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) case AARCH64_INSN_ADSB_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) insn = aarch64_insn_get_sub_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) case AARCH64_INSN_ADSB_ADD_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) insn = aarch64_insn_get_adds_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) case AARCH64_INSN_ADSB_SUB_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) insn = aarch64_insn_get_subs_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) /* We can't encode more than a 24bit value (12bit + 12bit shift) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (imm & ~(BIT(24) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) /* If we have something in the top 12 bits... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) if (imm & ~(SZ_4K - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) /* ... and in the low 12 bits -> error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (imm & (SZ_4K - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) imm >>= 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) insn |= AARCH64_INSN_LSL_12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) int immr, int imms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) enum aarch64_insn_bitfield_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) case AARCH64_INSN_BITFIELD_MOVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) insn = aarch64_insn_get_bfm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) insn = aarch64_insn_get_ubfm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) insn = aarch64_insn_get_sbfm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) mask = GENMASK(4, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) mask = GENMASK(5, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (immr & ~mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) pr_err("%s: invalid immr encoding %d\n", __func__, immr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (imms & ~mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) pr_err("%s: invalid imms encoding %d\n", __func__, imms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) int imm, int shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) enum aarch64_insn_movewide_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) case AARCH64_INSN_MOVEWIDE_ZERO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) insn = aarch64_insn_get_movz_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) case AARCH64_INSN_MOVEWIDE_KEEP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) insn = aarch64_insn_get_movk_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) case AARCH64_INSN_MOVEWIDE_INVERSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) insn = aarch64_insn_get_movn_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) pr_err("%s: unknown movewide encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) if (imm & ~(SZ_64K - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) if (shift != 0 && shift != 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) insn |= (shift >> 4) << 21;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) int shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) enum aarch64_insn_adsb_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) case AARCH64_INSN_ADSB_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) insn = aarch64_insn_get_add_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) case AARCH64_INSN_ADSB_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) insn = aarch64_insn_get_sub_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) case AARCH64_INSN_ADSB_ADD_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) insn = aarch64_insn_get_adds_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) case AARCH64_INSN_ADSB_SUB_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) insn = aarch64_insn_get_subs_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) if (shift & ~(SZ_32 - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) if (shift & ~(SZ_64 - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) enum aarch64_insn_data1_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) case AARCH64_INSN_DATA1_REVERSE_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) insn = aarch64_insn_get_rev16_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) case AARCH64_INSN_DATA1_REVERSE_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) insn = aarch64_insn_get_rev32_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) case AARCH64_INSN_DATA1_REVERSE_64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (variant != AARCH64_INSN_VARIANT_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) pr_err("%s: invalid variant for reverse64 %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) insn = aarch64_insn_get_rev64_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) pr_err("%s: unknown data1 encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) enum aarch64_insn_data2_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) case AARCH64_INSN_DATA2_UDIV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) insn = aarch64_insn_get_udiv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) case AARCH64_INSN_DATA2_SDIV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) insn = aarch64_insn_get_sdiv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) case AARCH64_INSN_DATA2_LSLV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) insn = aarch64_insn_get_lslv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) case AARCH64_INSN_DATA2_LSRV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) insn = aarch64_insn_get_lsrv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) case AARCH64_INSN_DATA2_ASRV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) insn = aarch64_insn_get_asrv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) case AARCH64_INSN_DATA2_RORV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) insn = aarch64_insn_get_rorv_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) pr_err("%s: unknown data2 encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) enum aarch64_insn_register reg1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) enum aarch64_insn_register reg2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) enum aarch64_insn_data3_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) case AARCH64_INSN_DATA3_MADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) insn = aarch64_insn_get_madd_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) case AARCH64_INSN_DATA3_MSUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) insn = aarch64_insn_get_msub_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) pr_err("%s: unknown data3 encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) reg1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) reg2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) int shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) enum aarch64_insn_logic_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) case AARCH64_INSN_LOGIC_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) insn = aarch64_insn_get_and_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) case AARCH64_INSN_LOGIC_BIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) insn = aarch64_insn_get_bic_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) case AARCH64_INSN_LOGIC_ORR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) insn = aarch64_insn_get_orr_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) case AARCH64_INSN_LOGIC_ORN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) insn = aarch64_insn_get_orn_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) case AARCH64_INSN_LOGIC_EOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) insn = aarch64_insn_get_eor_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) case AARCH64_INSN_LOGIC_EON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) insn = aarch64_insn_get_eon_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) case AARCH64_INSN_LOGIC_AND_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) insn = aarch64_insn_get_ands_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) insn = aarch64_insn_get_bics_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) pr_err("%s: unknown logical encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) if (shift & ~(SZ_32 - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (shift & ~(SZ_64 - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) pr_err("%s: invalid shift encoding %d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) * MOV (register) is architecturally an alias of ORR (shifted register) where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) enum aarch64_insn_register src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) enum aarch64_insn_variant variant)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) src, 0, variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) AARCH64_INSN_LOGIC_ORR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) enum aarch64_insn_register reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) enum aarch64_insn_adr_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) s32 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) case AARCH64_INSN_ADR_TYPE_ADR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) insn = aarch64_insn_get_adr_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) offset = addr - pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) case AARCH64_INSN_ADR_TYPE_ADRP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) insn = aarch64_insn_get_adrp_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) pr_err("%s: unknown adr encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (offset < -SZ_1M || offset >= SZ_1M)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) * Decode the imm field of a branch, and return the byte offset as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * signed value (so it can be used when computing a new branch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * target).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) s32 aarch64_get_branch_offset(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) s32 imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) return (imm << 6) >> 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) aarch64_insn_is_bcond(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) return (imm << 13) >> 11;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) return (imm << 18) >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) /* Unhandled instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * Encode the displacement of a branch in the imm field and return the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * updated instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) u32 aarch64_set_branch_offset(u32 insn, s32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) aarch64_insn_is_bcond(insn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) offset >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) /* Unhandled instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) s32 aarch64_insn_adrp_get_offset(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) BUG_ON(!aarch64_insn_is_adrp(insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) BUG_ON(!aarch64_insn_is_adrp(insn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) offset >> 12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) * Extract the Op/CR data from a msr/mrs instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) u32 aarch64_insn_extract_system_reg(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) return (insn & 0x1FFFE0) >> 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) bool aarch32_insn_is_wide(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) return insn >= 0xe800;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * Macros/defines for extracting register numbers from instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) return (insn & (0xf << offset)) >> offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) #define OPC2_MASK 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) #define OPC2_OFFSET 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) u32 aarch32_insn_mcr_extract_opc2(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) #define CRM_MASK 0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) u32 aarch32_insn_mcr_extract_crm(u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) return insn & CRM_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) static bool __kprobes __check_eq(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) return (pstate & PSR_Z_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) static bool __kprobes __check_ne(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) return (pstate & PSR_Z_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) static bool __kprobes __check_cs(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) return (pstate & PSR_C_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) static bool __kprobes __check_cc(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) return (pstate & PSR_C_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) static bool __kprobes __check_mi(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) return (pstate & PSR_N_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) static bool __kprobes __check_pl(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) return (pstate & PSR_N_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) static bool __kprobes __check_vs(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) return (pstate & PSR_V_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) static bool __kprobes __check_vc(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) return (pstate & PSR_V_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) static bool __kprobes __check_hi(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) return (pstate & PSR_C_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) static bool __kprobes __check_ls(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) return (pstate & PSR_C_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) static bool __kprobes __check_ge(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) return (pstate & PSR_N_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) static bool __kprobes __check_lt(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) return (pstate & PSR_N_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) static bool __kprobes __check_gt(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) /*PSR_N_BIT ^= PSR_V_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) unsigned long temp = pstate ^ (pstate << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) return (temp & PSR_N_BIT) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) static bool __kprobes __check_le(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) /*PSR_N_BIT ^= PSR_V_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) unsigned long temp = pstate ^ (pstate << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) return (temp & PSR_N_BIT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) static bool __kprobes __check_al(unsigned long pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) * it behaves identically to 0b1110 ("al").
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) pstate_check_t * const aarch32_opcode_cond_checks[16] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) __check_eq, __check_ne, __check_cs, __check_cc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) __check_mi, __check_pl, __check_vs, __check_vc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) __check_hi, __check_ls, __check_ge, __check_lt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) __check_gt, __check_le, __check_al, __check_al
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) static bool range_of_ones(u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) /* Doesn't handle full ones or full zeroes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) u64 sval = val >> __ffs64(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) /* One of Sean Eron Anderson's bithack tricks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return ((sval + 1) & (sval)) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) static u32 aarch64_encode_immediate(u64 imm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) unsigned int immr, imms, n, ones, ror, esz, tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) u64 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) esz = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) esz = 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) mask = GENMASK(esz - 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) /* Can't encode full zeroes, full ones, or value wider than the mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) if (!imm || imm == mask || imm & ~mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) * Inverse of Replicate(). Try to spot a repeating pattern
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) * with a pow2 stride.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) for (tmp = esz / 2; tmp >= 2; tmp /= 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) u64 emask = BIT(tmp) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) if ((imm & emask) != ((imm >> tmp) & emask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) esz = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) mask = emask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) /* N is only set if we're encoding a 64bit value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) n = esz == 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) /* Trim imm to the element size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) imm &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) /* That's how many ones we need to encode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) ones = hweight64(imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) * imms is set to (ones - 1), prefixed with a string of ones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) * and a zero if they fit. Cap it to 6 bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) imms = ones - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) imms |= 0xf << ffs(esz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) imms &= BIT(6) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) /* Compute the rotation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) if (range_of_ones(imm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * Pattern: 0..01..10..0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * Compute how many rotate we need to align it right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) ror = __ffs64(imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * Pattern: 0..01..10..01..1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) * Fill the unused top bits with ones, and check if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) * the result is a valid immediate (all ones with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) * contiguous ranges of zeroes).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) imm |= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (!range_of_ones(~imm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) * Compute the rotation to get a continuous set of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * ones, with the first bit set at position 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) ror = fls(~imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * immr is the number of bits we need to rotate back to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) * original set of ones. Note that this is relative to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) * element size...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) immr = (esz - ror) % esz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) enum aarch64_insn_register Rn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) enum aarch64_insn_register Rd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) u64 imm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) case AARCH64_INSN_LOGIC_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) insn = aarch64_insn_get_and_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) case AARCH64_INSN_LOGIC_ORR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) insn = aarch64_insn_get_orr_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) case AARCH64_INSN_LOGIC_EOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) insn = aarch64_insn_get_eor_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) case AARCH64_INSN_LOGIC_AND_SETFLAGS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) insn = aarch64_insn_get_ands_imm_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) pr_err("%s: unknown logical encoding %d\n", __func__, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) return aarch64_encode_immediate(imm, variant, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) enum aarch64_insn_register Rm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) enum aarch64_insn_register Rn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) enum aarch64_insn_register Rd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) u8 lsb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) insn = aarch64_insn_get_extr_value();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) switch (variant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) case AARCH64_INSN_VARIANT_32BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) if (lsb > 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) case AARCH64_INSN_VARIANT_64BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) if (lsb > 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) insn |= AARCH64_INSN_SF_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) pr_err("%s: unknown variant encoding %d\n", __func__, variant);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) return AARCH64_BREAK_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags