^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Just-In-Time compiler for eBPF filters on MIPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2017 Cavium, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Based on code from:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (c) 2014 Imagination Technologies Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Author: Markos Chandras <markos.chandras@imgtec.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/byteorder.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/cpu-features.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/isa-rev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/uasm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* Registers used by JIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define MIPS_R_ZERO 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define MIPS_R_AT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define MIPS_R_V0 2 /* BPF_R0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define MIPS_R_V1 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define MIPS_R_A0 4 /* BPF_R1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define MIPS_R_A1 5 /* BPF_R2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define MIPS_R_A2 6 /* BPF_R3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define MIPS_R_A3 7 /* BPF_R4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define MIPS_R_A4 8 /* BPF_R5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define MIPS_R_T4 12 /* BPF_AX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define MIPS_R_T5 13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define MIPS_R_T6 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define MIPS_R_T7 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define MIPS_R_S0 16 /* BPF_R6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define MIPS_R_S1 17 /* BPF_R7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define MIPS_R_S2 18 /* BPF_R8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define MIPS_R_S3 19 /* BPF_R9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define MIPS_R_S4 20 /* BPF_TCC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define MIPS_R_S5 21
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define MIPS_R_S6 22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define MIPS_R_S7 23
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define MIPS_R_T8 24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define MIPS_R_T9 25
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define MIPS_R_SP 29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define MIPS_R_RA 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* eBPF flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define EBPF_SAVE_S0 BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define EBPF_SAVE_S1 BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define EBPF_SAVE_S2 BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define EBPF_SAVE_S3 BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define EBPF_SAVE_S4 BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define EBPF_SAVE_RA BIT(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define EBPF_SEEN_FP BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define EBPF_SEEN_TC BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define EBPF_TCC_IN_V1 BIT(8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * For the mips64 ISA, we need to track the value range or type for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * each JIT register. The BPF machine requires zero extended 32-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * values, but the mips64 ISA requires sign extended 32-bit values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * At each point in the BPF program we track the state of every
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * register so that we can zero extend or sign extend as the BPF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * semantics require.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) enum reg_val_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* uninitialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) REG_UNKNOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /* not known to be 32-bit compatible. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) REG_64BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* 32-bit compatible, no truncation needed for 64-bit ops. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) REG_64BIT_32BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /* 32-bit compatible, need truncation for 64-bit ops. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) REG_32BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* 32-bit no sign/zero extension needed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) REG_32BIT_POS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * high bit of offsets indicates if long branch conversion done at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * this insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define OFFSETS_B_CONV BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * struct jit_ctx - JIT context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * @skf: The sk_filter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * @stack_size: eBPF stack size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * @idx: Instruction index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * @flags: JIT flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * @offsets: Instruction offsets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * @target: Memory location for the compiled filter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * @reg_val_types Packed enum reg_val_type for each register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct jit_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) const struct bpf_prog *skf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) int stack_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) u32 *offsets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) u32 *target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u64 *reg_val_types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned int long_b_conversion:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned int gen_b_offsets:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned int use_bbit_insns:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static void set_reg_val_type(u64 *rvt, int reg, enum reg_val_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) *rvt &= ~(7ull << (reg * 3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) *rvt |= ((u64)type << (reg * 3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) int index, int reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return (ctx->reg_val_types[index] >> (reg * 3)) & 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Simply emit the instruction if the JIT memory space has been allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define emit_instr_long(ctx, func64, func32, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if ((ctx)->target != NULL) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) u32 *p = &(ctx)->target[ctx->idx]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (IS_ENABLED(CONFIG_64BIT)) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) uasm_i_##func64(&p, ##__VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) else \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) uasm_i_##func32(&p, ##__VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) (ctx)->idx++; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define emit_instr(ctx, func, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) emit_instr_long(ctx, func, func, ##__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned long target_va, base_va;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) unsigned int r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (!ctx->target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) base_va = (unsigned long)ctx->target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) target_va = base_va + (ctx->offsets[target_idx] & ~OFFSETS_B_CONV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if ((base_va & ~0x0ffffffful) != (target_va & ~0x0ffffffful))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return (unsigned int)-1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) r = target_va & 0x0ffffffful;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* Compute the immediate value for PC-relative branches. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (!ctx->gen_b_offsets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * We want a pc-relative branch. tgt is the instruction offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * we want to jump to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * Branch on MIPS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * I: target_offset <- sign_extend(offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * I+1: PC += target_offset (delay slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * ctx->idx currently points to the branch instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * but the offset is added to the delay slot so we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * to subtract 4.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return (ctx->offsets[tgt] & ~OFFSETS_B_CONV) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) (ctx->idx * 4) - 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) enum which_ebpf_reg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) src_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) src_reg_no_fp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dst_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) dst_reg_fp_ok
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * For eBPF, the register mapping naturally falls out of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * requirements of eBPF and the MIPS n64 ABI. We don't maintain a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * separate frame pointer, so BPF_REG_10 relative accesses are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * adjusted to be $sp relative.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static int ebpf_to_mips_reg(struct jit_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) const struct bpf_insn *insn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) enum which_ebpf_reg w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int ebpf_reg = (w == src_reg || w == src_reg_no_fp) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) insn->src_reg : insn->dst_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) switch (ebpf_reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) case BPF_REG_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return MIPS_R_V0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) case BPF_REG_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return MIPS_R_A0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) case BPF_REG_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return MIPS_R_A1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) case BPF_REG_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return MIPS_R_A2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) case BPF_REG_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return MIPS_R_A3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) case BPF_REG_5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return MIPS_R_A4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) case BPF_REG_6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) ctx->flags |= EBPF_SAVE_S0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return MIPS_R_S0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) case BPF_REG_7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) ctx->flags |= EBPF_SAVE_S1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return MIPS_R_S1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) case BPF_REG_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) ctx->flags |= EBPF_SAVE_S2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return MIPS_R_S2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) case BPF_REG_9:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) ctx->flags |= EBPF_SAVE_S3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) return MIPS_R_S3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) case BPF_REG_10:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (w == dst_reg || w == src_reg_no_fp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) goto bad_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) ctx->flags |= EBPF_SEEN_FP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * Needs special handling, return something that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * cannot be clobbered just in case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) case BPF_REG_AX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return MIPS_R_T4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) bad_reg:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) WARN(1, "Illegal bpf reg: %d\n", ebpf_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * eBPF stack frame will be something like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * Entry $sp ------> +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * | $ra (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * | $s0 (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * | $s1 (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * | $s2 (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * | $s3 (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * | $s4 (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * | tmp-storage (if $ra saved) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * $sp + tmp_offset --> +--------------------------------+ <--BPF_REG_10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * | BPF_REG_10 relative storage |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * | MAX_BPF_STACK (optional) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * | . |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * | . |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * | . |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * $sp --------> +--------------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * If BPF_REG_10 is never referenced, then the MAX_BPF_STACK sized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * area is not allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static int gen_int_prologue(struct jit_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) int stack_adjust = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) int store_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) int locals_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (ctx->flags & EBPF_SAVE_RA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * If RA we are doing a function call and may need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * extra 8-byte tmp area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) stack_adjust += 2 * sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (ctx->flags & EBPF_SAVE_S0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) stack_adjust += sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (ctx->flags & EBPF_SAVE_S1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) stack_adjust += sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (ctx->flags & EBPF_SAVE_S2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) stack_adjust += sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (ctx->flags & EBPF_SAVE_S3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) stack_adjust += sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (ctx->flags & EBPF_SAVE_S4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) stack_adjust += sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) BUILD_BUG_ON(MAX_BPF_STACK & 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) stack_adjust += locals_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) ctx->stack_size = stack_adjust;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * First instruction initializes the tail call count (TCC).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * On tail call we skip this instruction, and the TCC is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * passed in $v1 from the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (stack_adjust)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) emit_instr_long(ctx, daddiu, addiu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) MIPS_R_SP, MIPS_R_SP, -stack_adjust);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) store_offset = stack_adjust - sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (ctx->flags & EBPF_SAVE_RA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) MIPS_R_RA, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (ctx->flags & EBPF_SAVE_S0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) MIPS_R_S0, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (ctx->flags & EBPF_SAVE_S1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) MIPS_R_S1, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (ctx->flags & EBPF_SAVE_S2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) MIPS_R_S2, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (ctx->flags & EBPF_SAVE_S3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) MIPS_R_S3, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (ctx->flags & EBPF_SAVE_S4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) emit_instr_long(ctx, sd, sw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) MIPS_R_S4, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) emit_instr_long(ctx, daddu, addu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) const struct bpf_prog *prog = ctx->skf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) int stack_adjust = ctx->stack_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) int store_offset = stack_adjust - sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) enum reg_val_type td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int r0 = MIPS_R_V0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (dest_reg == MIPS_R_RA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* Don't let zero extended value escape. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) td = get_reg_val_type(ctx, prog->len, BPF_REG_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (td == REG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) emit_instr(ctx, sll, r0, r0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (ctx->flags & EBPF_SAVE_RA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) MIPS_R_RA, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (ctx->flags & EBPF_SAVE_S0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) MIPS_R_S0, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (ctx->flags & EBPF_SAVE_S1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) MIPS_R_S1, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (ctx->flags & EBPF_SAVE_S2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) MIPS_R_S2, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (ctx->flags & EBPF_SAVE_S3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) MIPS_R_S3, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (ctx->flags & EBPF_SAVE_S4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) emit_instr_long(ctx, ld, lw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) MIPS_R_S4, store_offset, MIPS_R_SP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) store_offset -= sizeof(long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) emit_instr(ctx, jr, dest_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (stack_adjust)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) emit_instr_long(ctx, daddiu, addiu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) MIPS_R_SP, MIPS_R_SP, stack_adjust);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) static void gen_imm_to_reg(const struct bpf_insn *insn, int reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct jit_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (insn->imm >= S16_MIN && insn->imm <= S16_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) emit_instr(ctx, addiu, reg, MIPS_R_ZERO, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) int lower = (s16)(insn->imm & 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) int upper = insn->imm - lower;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) emit_instr(ctx, lui, reg, upper >> 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) emit_instr(ctx, addiu, reg, reg, lower);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) static int gen_imm_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) int upper_bound, lower_bound;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) int dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) switch (BPF_OP(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) case BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) case BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) upper_bound = S16_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) lower_bound = S16_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) case BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) upper_bound = -(int)S16_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) lower_bound = -(int)S16_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) case BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) case BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) case BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) upper_bound = 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) lower_bound = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) case BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) case BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) case BPF_ARSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Shift amounts are truncated, no need for bounds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) upper_bound = S32_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) lower_bound = S32_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * Immediate move clobbers the register, so no sign/zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * extension needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (BPF_CLASS(insn->code) == BPF_ALU64 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) BPF_OP(insn->code) != BPF_MOV &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) get_reg_val_type(ctx, idx, insn->dst_reg) == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* BPF_ALU | BPF_LSH doesn't need separate sign extension */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (BPF_CLASS(insn->code) == BPF_ALU &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) BPF_OP(insn->code) != BPF_LSH &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) BPF_OP(insn->code) != BPF_MOV &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) get_reg_val_type(ctx, idx, insn->dst_reg) != REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (insn->imm >= lower_bound && insn->imm <= upper_bound) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /* single insn immediate case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) switch (BPF_OP(insn->code) | BPF_CLASS(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) case BPF_ALU64 | BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) case BPF_ALU64 | BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) case BPF_ALU | BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) emit_instr(ctx, andi, dst, dst, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) case BPF_ALU64 | BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) case BPF_ALU | BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) emit_instr(ctx, ori, dst, dst, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) case BPF_ALU64 | BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) case BPF_ALU | BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) emit_instr(ctx, xori, dst, dst, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) case BPF_ALU64 | BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) emit_instr(ctx, daddiu, dst, dst, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) case BPF_ALU64 | BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) emit_instr(ctx, daddiu, dst, dst, -insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) case BPF_ALU64 | BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) emit_instr(ctx, dsrl_safe, dst, dst, insn->imm & 0x3f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) case BPF_ALU | BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) emit_instr(ctx, srl, dst, dst, insn->imm & 0x1f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) case BPF_ALU64 | BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) emit_instr(ctx, dsll_safe, dst, dst, insn->imm & 0x3f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) case BPF_ALU | BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) emit_instr(ctx, sll, dst, dst, insn->imm & 0x1f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) case BPF_ALU64 | BPF_ARSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) emit_instr(ctx, dsra_safe, dst, dst, insn->imm & 0x3f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) case BPF_ALU | BPF_ARSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) emit_instr(ctx, sra, dst, dst, insn->imm & 0x1f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) case BPF_ALU | BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) emit_instr(ctx, addiu, dst, MIPS_R_ZERO, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) case BPF_ALU | BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) emit_instr(ctx, addiu, dst, dst, insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) case BPF_ALU | BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) emit_instr(ctx, addiu, dst, dst, -insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /* multi insn immediate case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (BPF_OP(insn->code) == BPF_MOV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) gen_imm_to_reg(insn, dst, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) switch (BPF_OP(insn->code) | BPF_CLASS(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) case BPF_ALU64 | BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) case BPF_ALU | BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) emit_instr(ctx, and, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) case BPF_ALU64 | BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) case BPF_ALU | BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) emit_instr(ctx, or, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) case BPF_ALU64 | BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) case BPF_ALU | BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) emit_instr(ctx, xor, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) case BPF_ALU64 | BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) emit_instr(ctx, daddu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) case BPF_ALU64 | BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) emit_instr(ctx, dsubu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) case BPF_ALU | BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) emit_instr(ctx, addu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) case BPF_ALU | BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) emit_instr(ctx, subu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) static void emit_const_to_reg(struct jit_ctx *ctx, int dst, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (value >= 0xffffffffffff8000ull || value < 0x8000ull) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, (int)value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) } else if (value >= 0xffffffff80000000ull ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) (value < 0x80000000 && value > 0xffff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) emit_instr(ctx, lui, dst, (s32)(s16)(value >> 16));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) emit_instr(ctx, ori, dst, dst, (unsigned int)(value & 0xffff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) bool seen_part = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) int needed_shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) for (i = 0; i < 4; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) u64 part = (value >> (16 * (3 - i))) & 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (seen_part && needed_shift > 0 && (part || i == 3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) emit_instr(ctx, dsll_safe, dst, dst, needed_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) needed_shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (part) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (i == 0 || (!seen_part && i < 3 && part < 0x8000)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) emit_instr(ctx, lui, dst, (s32)(s16)part);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) needed_shift = -16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) emit_instr(ctx, ori, dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) seen_part ? dst : MIPS_R_ZERO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) (unsigned int)part);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) seen_part = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (seen_part)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) needed_shift += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static int emit_bpf_tail_call(struct jit_ctx *ctx, int this_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int off, b_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) int tcc_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) ctx->flags |= EBPF_SEEN_TC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * if (index >= array->map.max_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) off = offsetof(struct bpf_array, map.max_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) emit_instr(ctx, lwu, MIPS_R_T5, off, MIPS_R_A1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) emit_instr(ctx, sltu, MIPS_R_AT, MIPS_R_T5, MIPS_R_A2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) b_off = b_imm(this_idx + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) emit_instr(ctx, bne, MIPS_R_AT, MIPS_R_ZERO, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * if (TCC-- < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /* Delay slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) tcc_reg = (ctx->flags & EBPF_TCC_IN_V1) ? MIPS_R_V1 : MIPS_R_S4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) emit_instr(ctx, daddiu, MIPS_R_T5, tcc_reg, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) b_off = b_imm(this_idx + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) emit_instr(ctx, bltz, tcc_reg, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * prog = array->ptrs[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * if (prog == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) /* Delay slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) emit_instr(ctx, dsll, MIPS_R_T8, MIPS_R_A2, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, MIPS_R_A1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) off = offsetof(struct bpf_array, ptrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) emit_instr(ctx, ld, MIPS_R_AT, off, MIPS_R_T8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) b_off = b_imm(this_idx + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) emit_instr(ctx, beq, MIPS_R_AT, MIPS_R_ZERO, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /* Delay slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* goto *(prog->bpf_func + 4); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) off = offsetof(struct bpf_prog, bpf_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) emit_instr(ctx, ld, MIPS_R_T9, off, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /* All systems are go... propagate TCC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) emit_instr(ctx, daddu, MIPS_R_V1, MIPS_R_T5, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) /* Skip first instruction (TCC initialization) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) emit_instr(ctx, daddiu, MIPS_R_T9, MIPS_R_T9, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return build_int_epilogue(ctx, MIPS_R_T9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) static bool is_bad_offset(int b_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return b_off > 0x1ffff || b_off < -0x20000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* Returns the number of insn slots consumed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) int this_idx, int exit_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) int src, dst, r, td, ts, mem_off, b_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) bool need_swap, did_move, cmp_eq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) unsigned int target = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) u64 t64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) s64 t64s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) int bpf_op = BPF_OP(insn->code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (IS_ENABLED(CONFIG_32BIT) && ((BPF_CLASS(insn->code) == BPF_ALU64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) || (bpf_op == BPF_DW)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) switch (insn->code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) case BPF_ALU64 | BPF_ADD | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) case BPF_ALU64 | BPF_SUB | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) case BPF_ALU64 | BPF_OR | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) case BPF_ALU64 | BPF_AND | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) case BPF_ALU64 | BPF_LSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) case BPF_ALU64 | BPF_RSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) case BPF_ALU64 | BPF_XOR | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) case BPF_ALU64 | BPF_ARSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) case BPF_ALU64 | BPF_MOV | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) case BPF_ALU | BPF_MOV | BPF_K: /* ALU32_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) case BPF_ALU | BPF_ADD | BPF_K: /* ALU32_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) case BPF_ALU | BPF_SUB | BPF_K: /* ALU32_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) case BPF_ALU | BPF_OR | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) case BPF_ALU | BPF_AND | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) case BPF_ALU | BPF_LSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) case BPF_ALU | BPF_RSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) case BPF_ALU | BPF_XOR | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) case BPF_ALU | BPF_ARSH | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) r = gen_imm_insn(insn, ctx, this_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) case BPF_ALU64 | BPF_MUL | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) if (insn->imm == 1) /* Mult by 1 is a nop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) emit_instr(ctx, dmulu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) emit_instr(ctx, dmultu, MIPS_R_AT, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) case BPF_ALU64 | BPF_NEG | BPF_K: /* ALU64_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) emit_instr(ctx, dsubu, dst, MIPS_R_ZERO, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) case BPF_ALU | BPF_MUL | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (td == REG_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (insn->imm == 1) /* Mult by 1 is a nop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) emit_instr(ctx, mulu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) emit_instr(ctx, multu, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) case BPF_ALU | BPF_NEG | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) if (td == REG_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) emit_instr(ctx, subu, dst, MIPS_R_ZERO, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) case BPF_ALU | BPF_DIV | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) case BPF_ALU | BPF_MOD | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (insn->imm == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (td == REG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (insn->imm == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) /* div by 1 is a nop, mod by 1 is zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (bpf_op == BPF_MOD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) emit_instr(ctx, divu_r6, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) emit_instr(ctx, divu, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) emit_instr(ctx, mfhi, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) case BPF_ALU64 | BPF_DIV | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) case BPF_ALU64 | BPF_MOD | BPF_K: /* ALU_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (insn->imm == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (insn->imm == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) /* div by 1 is a nop, mod by 1 is zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (bpf_op == BPF_MOD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) emit_instr(ctx, ddivu_r6, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) emit_instr(ctx, ddivu, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) emit_instr(ctx, mfhi, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) case BPF_ALU64 | BPF_MOV | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) case BPF_ALU64 | BPF_ADD | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) case BPF_ALU64 | BPF_SUB | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) case BPF_ALU64 | BPF_XOR | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) case BPF_ALU64 | BPF_OR | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) case BPF_ALU64 | BPF_AND | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) case BPF_ALU64 | BPF_MUL | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) case BPF_ALU64 | BPF_DIV | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) case BPF_ALU64 | BPF_MOD | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) case BPF_ALU64 | BPF_LSH | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) case BPF_ALU64 | BPF_RSH | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) case BPF_ALU64 | BPF_ARSH | BPF_X: /* ALU64_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) src = ebpf_to_mips_reg(ctx, insn, src_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (src < 0 || dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) did_move = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) if (insn->src_reg == BPF_REG_10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (bpf_op == BPF_MOV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) emit_instr(ctx, daddiu, dst, MIPS_R_SP, MAX_BPF_STACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) did_move = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) emit_instr(ctx, daddiu, MIPS_R_AT, MIPS_R_SP, MAX_BPF_STACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) } else if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) int tmp_reg = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) if (bpf_op == BPF_MOV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) tmp_reg = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) did_move = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) emit_instr(ctx, daddu, tmp_reg, src, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) emit_instr(ctx, dinsu, tmp_reg, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) switch (bpf_op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) case BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (!did_move)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) emit_instr(ctx, daddu, dst, src, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) case BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) emit_instr(ctx, daddu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) case BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) emit_instr(ctx, dsubu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) case BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) emit_instr(ctx, xor, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) case BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) emit_instr(ctx, or, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) case BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) emit_instr(ctx, and, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) case BPF_MUL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) emit_instr(ctx, dmulu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) emit_instr(ctx, dmultu, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) case BPF_DIV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) case BPF_MOD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) emit_instr(ctx, ddivu_r6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) emit_instr(ctx, modu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) emit_instr(ctx, ddivu, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) emit_instr(ctx, mfhi, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) case BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) emit_instr(ctx, dsllv, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) case BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) emit_instr(ctx, dsrlv, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) case BPF_ARSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) emit_instr(ctx, dsrav, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) pr_err("ALU64_REG NOT HANDLED\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) case BPF_ALU | BPF_MOV | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) case BPF_ALU | BPF_ADD | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) case BPF_ALU | BPF_SUB | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) case BPF_ALU | BPF_XOR | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) case BPF_ALU | BPF_OR | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) case BPF_ALU | BPF_AND | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) case BPF_ALU | BPF_MUL | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) case BPF_ALU | BPF_DIV | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) case BPF_ALU | BPF_MOD | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) case BPF_ALU | BPF_LSH | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) case BPF_ALU | BPF_RSH | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) case BPF_ALU | BPF_ARSH | BPF_X: /* ALU_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (src < 0 || dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) if (td == REG_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) did_move = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) if (ts == REG_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) int tmp_reg = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (bpf_op == BPF_MOV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) tmp_reg = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) did_move = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) emit_instr(ctx, sll, tmp_reg, src, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) switch (bpf_op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) case BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) if (!did_move)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) emit_instr(ctx, addu, dst, src, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) case BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) emit_instr(ctx, addu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) case BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) emit_instr(ctx, subu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) case BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) emit_instr(ctx, xor, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) case BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) emit_instr(ctx, or, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) case BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) emit_instr(ctx, and, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) case BPF_MUL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) emit_instr(ctx, mul, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) case BPF_DIV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) case BPF_MOD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) emit_instr(ctx, divu_r6, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) emit_instr(ctx, modu, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) emit_instr(ctx, divu, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) if (bpf_op == BPF_DIV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) emit_instr(ctx, mflo, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) emit_instr(ctx, mfhi, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) case BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) emit_instr(ctx, sllv, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) case BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) emit_instr(ctx, srlv, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) case BPF_ARSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) emit_instr(ctx, srav, dst, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) pr_err("ALU_REG NOT HANDLED\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) case BPF_JMP | BPF_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (this_idx + 1 < exit_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) b_off = b_imm(exit_idx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) emit_instr(ctx, beq, MIPS_R_ZERO, MIPS_R_ZERO, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) case BPF_JMP | BPF_JEQ | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) case BPF_JMP | BPF_JNE | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) cmp_eq = (bpf_op == BPF_JEQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (insn->imm == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) src = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) goto jeq_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) case BPF_JMP | BPF_JEQ | BPF_X: /* JMP_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) case BPF_JMP | BPF_JNE | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) case BPF_JMP | BPF_JSLT | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) case BPF_JMP | BPF_JSLE | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) case BPF_JMP | BPF_JSGT | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) case BPF_JMP | BPF_JSGE | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) case BPF_JMP | BPF_JLT | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) case BPF_JMP | BPF_JLE | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) case BPF_JMP | BPF_JGT | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) case BPF_JMP | BPF_JGE | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) case BPF_JMP | BPF_JSET | BPF_X:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (src < 0 || dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (td == REG_32BIT && ts != REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) emit_instr(ctx, sll, MIPS_R_AT, src, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) } else if (ts == REG_32BIT && td != REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) emit_instr(ctx, sll, MIPS_R_AT, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) dst = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) if (bpf_op == BPF_JSET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) emit_instr(ctx, and, MIPS_R_AT, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) cmp_eq = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) dst = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) src = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) } else if (bpf_op == BPF_JSGT || bpf_op == BPF_JSLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) emit_instr(ctx, dsubu, MIPS_R_AT, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) b_off = b_imm(exit_idx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (bpf_op == BPF_JSGT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) emit_instr(ctx, blez, MIPS_R_AT, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) return 2; /* We consumed the exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) b_off = b_imm(this_idx + insn->off + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) if (bpf_op == BPF_JSGT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) emit_instr(ctx, blez, MIPS_R_AT, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) } else if (bpf_op == BPF_JSGE || bpf_op == BPF_JSLT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) emit_instr(ctx, slt, MIPS_R_AT, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) cmp_eq = bpf_op == BPF_JSGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) dst = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) src = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) } else if (bpf_op == BPF_JGT || bpf_op == BPF_JLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) /* dst or src could be AT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) emit_instr(ctx, dsubu, MIPS_R_T8, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) /* SP known to be non-zero, movz becomes boolean not */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (MIPS_ISA_REV >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) emit_instr(ctx, seleqz, MIPS_R_T9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) MIPS_R_SP, MIPS_R_T8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) emit_instr(ctx, movz, MIPS_R_T9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) MIPS_R_SP, MIPS_R_T8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) emit_instr(ctx, movn, MIPS_R_T9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) MIPS_R_ZERO, MIPS_R_T8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) cmp_eq = bpf_op == BPF_JGT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) dst = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) src = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) } else if (bpf_op == BPF_JGE || bpf_op == BPF_JLT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) cmp_eq = bpf_op == BPF_JGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) dst = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) src = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) } else { /* JNE/JEQ case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) cmp_eq = (bpf_op == BPF_JEQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) jeq_common:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * If the next insn is EXIT and we are jumping arround
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * only it, invert the sense of the compare and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * conditionally jump to the exit. Poor man's branch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * chaining.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) b_off = b_imm(exit_idx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (is_bad_offset(b_off)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) target = j_target(ctx, exit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) if (target == (unsigned int)-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) cmp_eq = !cmp_eq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) b_off = 4 * 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) ctx->offsets[this_idx] |= OFFSETS_B_CONV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) ctx->long_b_conversion = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) if (cmp_eq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) emit_instr(ctx, bne, dst, src, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) emit_instr(ctx, beq, dst, src, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) if (ctx->offsets[this_idx] & OFFSETS_B_CONV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) emit_instr(ctx, j, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) return 2; /* We consumed the exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) b_off = b_imm(this_idx + insn->off + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (is_bad_offset(b_off)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) target = j_target(ctx, this_idx + insn->off + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (target == (unsigned int)-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) cmp_eq = !cmp_eq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) b_off = 4 * 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) ctx->offsets[this_idx] |= OFFSETS_B_CONV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) ctx->long_b_conversion = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) if (cmp_eq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) emit_instr(ctx, beq, dst, src, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) emit_instr(ctx, bne, dst, src, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) if (ctx->offsets[this_idx] & OFFSETS_B_CONV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) emit_instr(ctx, j, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) case BPF_JMP | BPF_JSGT | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) case BPF_JMP | BPF_JSGE | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) case BPF_JMP | BPF_JSLT | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) case BPF_JMP | BPF_JSLE | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) cmp_eq = (bpf_op == BPF_JSGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (insn->imm == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) b_off = b_imm(exit_idx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) switch (bpf_op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) case BPF_JSGT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) emit_instr(ctx, blez, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) case BPF_JSGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) emit_instr(ctx, bltz, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) case BPF_JSLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) emit_instr(ctx, bgez, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) case BPF_JSLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) emit_instr(ctx, bgtz, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) return 2; /* We consumed the exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) b_off = b_imm(this_idx + insn->off + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) switch (bpf_op) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) case BPF_JSGT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) emit_instr(ctx, bgtz, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) case BPF_JSGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) emit_instr(ctx, bgez, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) case BPF_JSLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) emit_instr(ctx, bltz, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) case BPF_JSLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) emit_instr(ctx, blez, dst, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * only "LT" compare available, so we must use imm + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * to generate "GT" and imm -1 to generate LE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) if (bpf_op == BPF_JSGT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) t64s = insn->imm + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) else if (bpf_op == BPF_JSLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) t64s = insn->imm + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) t64s = insn->imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) cmp_eq = bpf_op == BPF_JSGT || bpf_op == BPF_JSGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (t64s >= S16_MIN && t64s <= S16_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) emit_instr(ctx, slti, MIPS_R_AT, dst, (int)t64s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) dst = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) goto jeq_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) emit_instr(ctx, slt, MIPS_R_AT, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) dst = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) goto jeq_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) case BPF_JMP | BPF_JGT | BPF_K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) case BPF_JMP | BPF_JGE | BPF_K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) case BPF_JMP | BPF_JLT | BPF_K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) case BPF_JMP | BPF_JLE | BPF_K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) cmp_eq = (bpf_op == BPF_JGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * only "LT" compare available, so we must use imm + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * to generate "GT" and imm -1 to generate LE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) if (bpf_op == BPF_JGT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) t64s = (u64)(u32)(insn->imm) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) else if (bpf_op == BPF_JLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) t64s = (u64)(u32)(insn->imm) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) t64s = (u64)(u32)(insn->imm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) cmp_eq = bpf_op == BPF_JGT || bpf_op == BPF_JGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) emit_instr(ctx, sltu, MIPS_R_AT, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) dst = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) goto jeq_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) case BPF_JMP | BPF_JSET | BPF_K: /* JMP_IMM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (ctx->use_bbit_insns && hweight32((u32)insn->imm) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) if ((insn + 1)->code == (BPF_JMP | BPF_EXIT) && insn->off == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) b_off = b_imm(exit_idx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) emit_instr(ctx, bbit0, dst, ffs((u32)insn->imm) - 1, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) return 2; /* We consumed the exit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) b_off = b_imm(this_idx + insn->off + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (is_bad_offset(b_off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) emit_instr(ctx, bbit1, dst, ffs((u32)insn->imm) - 1, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) t64 = (u32)insn->imm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) emit_const_to_reg(ctx, MIPS_R_AT, t64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) emit_instr(ctx, and, MIPS_R_AT, dst, MIPS_R_AT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) dst = MIPS_R_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) cmp_eq = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) goto jeq_common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) case BPF_JMP | BPF_JA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) * Prefer relative branch for easier debugging, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) * fall back if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) b_off = b_imm(this_idx + insn->off + 1, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) if (is_bad_offset(b_off)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) target = j_target(ctx, this_idx + insn->off + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) if (target == (unsigned int)-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) return -E2BIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) emit_instr(ctx, j, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) emit_instr(ctx, b, b_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) case BPF_LD | BPF_DW | BPF_IMM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) if (insn->src_reg != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) t64 = ((u64)(u32)insn->imm) | ((u64)(insn + 1)->imm << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) emit_const_to_reg(ctx, dst, t64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) return 2; /* Double slot insn */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) case BPF_JMP | BPF_CALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) ctx->flags |= EBPF_SAVE_RA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) t64s = (s64)insn->imm + (long)__bpf_call_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) /* delay slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) case BPF_JMP | BPF_TAIL_CALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) if (emit_bpf_tail_call(ctx, this_idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) case BPF_ALU | BPF_END | BPF_FROM_BE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) case BPF_ALU | BPF_END | BPF_FROM_LE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (insn->imm == 64 && td == REG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (insn->imm != 64 && td == REG_64BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) /* sign extend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) emit_instr(ctx, sll, dst, dst, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) #ifdef __BIG_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) need_swap = (BPF_SRC(insn->code) == BPF_FROM_LE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) need_swap = (BPF_SRC(insn->code) == BPF_FROM_BE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) if (insn->imm == 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) if (need_swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) emit_instr(ctx, wsbh, dst, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) emit_instr(ctx, andi, dst, dst, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) } else if (insn->imm == 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) if (need_swap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) emit_instr(ctx, wsbh, dst, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) emit_instr(ctx, rotr, dst, dst, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) } else { /* 64-bit*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) if (need_swap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) emit_instr(ctx, dsbh, dst, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) emit_instr(ctx, dshd, dst, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) case BPF_ST | BPF_NOSPEC: /* speculation barrier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) case BPF_ST | BPF_B | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) case BPF_ST | BPF_H | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) case BPF_ST | BPF_W | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) case BPF_ST | BPF_DW | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) if (insn->dst_reg == BPF_REG_10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) ctx->flags |= EBPF_SEEN_FP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) dst = MIPS_R_SP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) mem_off = insn->off + MAX_BPF_STACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) mem_off = insn->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) gen_imm_to_reg(insn, MIPS_R_AT, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) case BPF_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) emit_instr(ctx, sb, MIPS_R_AT, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) case BPF_H:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) emit_instr(ctx, sh, MIPS_R_AT, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) emit_instr(ctx, sw, MIPS_R_AT, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) emit_instr(ctx, sd, MIPS_R_AT, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) case BPF_LDX | BPF_B | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) case BPF_LDX | BPF_H | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) case BPF_LDX | BPF_W | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) case BPF_LDX | BPF_DW | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) if (insn->src_reg == BPF_REG_10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ctx->flags |= EBPF_SEEN_FP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) src = MIPS_R_SP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) mem_off = insn->off + MAX_BPF_STACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) if (src < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) return src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) mem_off = insn->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) case BPF_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) emit_instr(ctx, lbu, dst, mem_off, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) case BPF_H:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) emit_instr(ctx, lhu, dst, mem_off, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) emit_instr(ctx, lw, dst, mem_off, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) emit_instr(ctx, ld, dst, mem_off, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) case BPF_STX | BPF_B | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) case BPF_STX | BPF_H | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) case BPF_STX | BPF_W | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) case BPF_STX | BPF_DW | BPF_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) case BPF_STX | BPF_W | BPF_XADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) case BPF_STX | BPF_DW | BPF_XADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (insn->dst_reg == BPF_REG_10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) ctx->flags |= EBPF_SEEN_FP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) dst = MIPS_R_SP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) mem_off = insn->off + MAX_BPF_STACK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (dst < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) mem_off = insn->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) if (src < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) return src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (BPF_MODE(insn->code) == BPF_XADD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) * If mem_off does not fit within the 9 bit ll/sc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * instruction immediate field, use a temp reg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) if (MIPS_ISA_REV >= 6 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) (mem_off >= BIT(8) || mem_off < -BIT(8))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) emit_instr(ctx, daddiu, MIPS_R_T6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) dst, mem_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) mem_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) dst = MIPS_R_T6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) emit_instr(ctx, sll, MIPS_R_AT, src, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) emit_instr(ctx, ll, MIPS_R_T8, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) emit_instr(ctx, addu, MIPS_R_T8, MIPS_R_T8, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) emit_instr(ctx, sc, MIPS_R_T8, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) * On failure back up to LL (-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) * instructions of 4 bytes each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) emit_instr(ctx, lld, MIPS_R_T8, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) emit_instr(ctx, scd, MIPS_R_T8, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) emit_instr(ctx, nop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) } else { /* BPF_MEM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) case BPF_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) emit_instr(ctx, sb, src, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) case BPF_H:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) emit_instr(ctx, sh, src, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) emit_instr(ctx, sw, src, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) src = MIPS_R_AT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) emit_instr(ctx, sd, src, mem_off, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) pr_err("NOT HANDLED %d - (%02x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) this_idx, (unsigned int)insn->code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) #define RVT_VISITED_MASK 0xc000000000000000ull
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) #define RVT_FALL_THROUGH 0x4000000000000000ull
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) #define RVT_BRANCH_TAKEN 0x8000000000000000ull
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) #define RVT_DONE (RVT_FALL_THROUGH | RVT_BRANCH_TAKEN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) static int build_int_body(struct jit_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) const struct bpf_prog *prog = ctx->skf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) const struct bpf_insn *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) int i, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) for (i = 0; i < prog->len; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) insn = prog->insnsi + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if ((ctx->reg_val_types[i] & RVT_VISITED_MASK) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) /* dead instruction, don't emit it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) if (ctx->target == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) ctx->offsets[i] = (ctx->offsets[i] & OFFSETS_B_CONV) | (ctx->idx * 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) r = build_one_insn(insn, ctx, i, prog->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (r < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) return r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) i += r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) /* epilogue offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) if (ctx->target == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) ctx->offsets[i] = ctx->idx * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * All exits have an offset of the epilogue, some offsets may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * not have been set due to banch-around threading, so set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * them now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) if (ctx->target == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) for (i = 0; i < prog->len; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) insn = prog->insnsi + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) if (insn->code == (BPF_JMP | BPF_EXIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) ctx->offsets[i] = ctx->idx * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) /* return the last idx processed, or negative for error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) static int reg_val_propagate_range(struct jit_ctx *ctx, u64 initial_rvt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) int start_idx, bool follow_taken)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) const struct bpf_prog *prog = ctx->skf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) const struct bpf_insn *insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) u64 exit_rvt = initial_rvt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) u64 *rvt = ctx->reg_val_types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) int reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) for (idx = start_idx; idx < prog->len; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) rvt[idx] = (rvt[idx] & RVT_VISITED_MASK) | exit_rvt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) insn = prog->insnsi + idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) switch (BPF_CLASS(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) case BPF_ALU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) switch (BPF_OP(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) case BPF_ADD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) case BPF_SUB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) case BPF_MUL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) case BPF_DIV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) case BPF_OR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) case BPF_AND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) case BPF_LSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) case BPF_RSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) case BPF_NEG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) case BPF_MOD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) case BPF_XOR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) case BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) if (BPF_SRC(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) /* IMM to REG move*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) if (insn->imm >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) case BPF_END:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (insn->imm == 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) else if (insn->imm == 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) else /* insn->imm == 16 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) case BPF_ALU64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) switch (BPF_OP(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) case BPF_MOV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (BPF_SRC(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) /* REG to REG move*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) /* IMM to REG move*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) if (insn->imm >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) case BPF_LD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) if (BPF_MODE(insn->code) == BPF_IMM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) s64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) val = (s64)((u32)insn->imm | ((u64)(insn + 1)->imm << 32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (val > 0 && val <= S32_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) else if (val >= S32_MIN && val <= S32_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) case BPF_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) case BPF_H:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) if (BPF_MODE(insn->code) == BPF_IMM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) set_reg_val_type(&exit_rvt, insn->dst_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) insn->imm >= 0 ? REG_32BIT_POS : REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) case BPF_LDX:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) switch (BPF_SIZE(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) case BPF_DW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) case BPF_B:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) case BPF_H:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT_POS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) case BPF_W:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) set_reg_val_type(&exit_rvt, insn->dst_reg, REG_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) case BPF_JMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) switch (BPF_OP(insn->code)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) case BPF_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) rvt[idx] = RVT_DONE | exit_rvt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) rvt[prog->len] = exit_rvt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) case BPF_JA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) idx += insn->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) case BPF_JEQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) case BPF_JGT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) case BPF_JGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) case BPF_JLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) case BPF_JLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) case BPF_JSET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) case BPF_JNE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) case BPF_JSGT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) case BPF_JSGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) case BPF_JSLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) case BPF_JSLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) if (follow_taken) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) rvt[idx] |= RVT_BRANCH_TAKEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) idx += insn->off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) follow_taken = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) rvt[idx] |= RVT_FALL_THROUGH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) case BPF_CALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) set_reg_val_type(&exit_rvt, BPF_REG_0, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) /* Upon call return, argument registers are clobbered. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) for (reg = BPF_REG_0; reg <= BPF_REG_5; reg++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) set_reg_val_type(&exit_rvt, reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) WARN(1, "Unhandled BPF_JMP case.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) rvt[idx] |= RVT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) * Track the value range (i.e. 32-bit vs. 64-bit) of each register at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) * each eBPF insn. This allows unneeded sign and zero extension
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) * operations to be omitted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) * Doesn't handle yet confluence of control paths with conflicting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) * ranges, but it is good enough for most sane code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) static int reg_val_propagate(struct jit_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) const struct bpf_prog *prog = ctx->skf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) u64 exit_rvt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) int reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * 11 registers * 3 bits/reg leaves top bits free for other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) * uses. Bit-62..63 used to see if we have visited an insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) exit_rvt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) /* Upon entry, argument registers are 64-bit. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) for (reg = BPF_REG_1; reg <= BPF_REG_5; reg++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) set_reg_val_type(&exit_rvt, reg, REG_64BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) * First follow all conditional branches on the fall-through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) * edge of control flow..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) reg_val_propagate_range(ctx, exit_rvt, 0, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) restart_search:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) * Then repeatedly find the first conditional branch where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) * both edges of control flow have not been taken, and follow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) * the branch taken edge. We will end up restarting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) * search once per conditional branch insn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) for (i = 0; i < prog->len; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) u64 rvt = ctx->reg_val_types[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) if ((rvt & RVT_VISITED_MASK) == RVT_DONE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) (rvt & RVT_VISITED_MASK) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) if ((rvt & RVT_VISITED_MASK) == RVT_FALL_THROUGH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) } else { /* RVT_BRANCH_TAKEN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) WARN(1, "Unexpected RVT_BRANCH_TAKEN case.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) goto restart_search;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) * Eventually all conditional branches have been followed on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) * both branches and we are done. Any insn that has not been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) * visited at this point is dead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) static void jit_fill_hole(void *area, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) u32 *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) /* We are guaranteed to have aligned memory. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) for (p = area; size >= sizeof(u32); size -= sizeof(u32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) uasm_i_break(&p, BRK_BUG); /* Increments p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) struct bpf_prog *orig_prog = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) bool tmp_blinded = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) struct bpf_prog *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) struct bpf_binary_header *header = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) struct jit_ctx ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) unsigned int image_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) u8 *image_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) if (!prog->jit_requested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) return prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) tmp = bpf_jit_blind_constants(prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) /* If blinding was requested and we failed during blinding,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) * we must fall back to the interpreter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) if (IS_ERR(tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) return orig_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) if (tmp != prog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) tmp_blinded = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) prog = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) memset(&ctx, 0, sizeof(ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) switch (current_cpu_type()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) case CPU_CAVIUM_OCTEON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) case CPU_CAVIUM_OCTEON_PLUS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) case CPU_CAVIUM_OCTEON2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) case CPU_CAVIUM_OCTEON3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) ctx.use_bbit_insns = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) ctx.use_bbit_insns = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) ctx.offsets = kcalloc(prog->len + 1, sizeof(*ctx.offsets), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) if (ctx.offsets == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) ctx.reg_val_types = kcalloc(prog->len + 1, sizeof(*ctx.reg_val_types), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) if (ctx.reg_val_types == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) ctx.skf = prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) if (reg_val_propagate(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) * First pass discovers used resources and instruction offsets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * assuming short branches are used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) if (build_int_body(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) * If no calls are made (EBPF_SAVE_RA), then tail call count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) * in $v1, else we must save in n$s4.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) if (ctx.flags & EBPF_SEEN_TC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) if (ctx.flags & EBPF_SAVE_RA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) ctx.flags |= EBPF_SAVE_S4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) ctx.flags |= EBPF_TCC_IN_V1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) * Second pass generates offsets, if any branches are out of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) * range a jump-around long sequence is generated, and we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) * to try again from the beginning to generate the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * offsets. This is done until no additional conversions are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) * necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) ctx.idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) ctx.gen_b_offsets = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) ctx.long_b_conversion = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) if (gen_int_prologue(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) if (build_int_body(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (build_int_epilogue(&ctx, MIPS_R_RA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) } while (ctx.long_b_conversion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) image_size = 4 * ctx.idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) header = bpf_jit_binary_alloc(image_size, &image_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) sizeof(u32), jit_fill_hole);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) if (header == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) ctx.target = (u32 *)image_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) /* Third pass generates the code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) ctx.idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) if (gen_int_prologue(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) if (build_int_body(&ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) if (build_int_epilogue(&ctx, MIPS_R_RA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) goto out_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) /* Update the icache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) flush_icache_range((unsigned long)ctx.target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) (unsigned long)&ctx.target[ctx.idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) if (bpf_jit_enable > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) /* Dump JIT code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) bpf_jit_dump(prog->len, image_size, 2, ctx.target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) bpf_jit_binary_lock_ro(header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) prog->bpf_func = (void *)ctx.target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) prog->jited = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) prog->jited_len = image_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) out_normal:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) if (tmp_blinded)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) bpf_jit_prog_release_other(prog, prog == orig_prog ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) tmp : orig_prog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) kfree(ctx.offsets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) kfree(ctx.reg_val_types);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) return prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) out_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) prog = orig_prog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) if (header)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) bpf_jit_binary_free(header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) goto out_normal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags