Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Copyright (C) 2020 Western Digital Corporation or its affiliates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/csr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #define INSN_MATCH_LB			0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #define INSN_MASK_LB			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define INSN_MATCH_LH			0x1003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #define INSN_MASK_LH			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define INSN_MATCH_LW			0x2003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define INSN_MASK_LW			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define INSN_MATCH_LD			0x3003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define INSN_MASK_LD			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #define INSN_MATCH_LBU			0x4003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define INSN_MASK_LBU			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define INSN_MATCH_LHU			0x5003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #define INSN_MASK_LHU			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #define INSN_MATCH_LWU			0x6003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #define INSN_MASK_LWU			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #define INSN_MATCH_SB			0x23
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #define INSN_MASK_SB			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #define INSN_MATCH_SH			0x1023
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #define INSN_MASK_SH			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #define INSN_MATCH_SW			0x2023
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #define INSN_MASK_SW			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #define INSN_MATCH_SD			0x3023
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #define INSN_MASK_SD			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #define INSN_MATCH_FLW			0x2007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #define INSN_MASK_FLW			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #define INSN_MATCH_FLD			0x3007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #define INSN_MASK_FLD			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #define INSN_MATCH_FLQ			0x4007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #define INSN_MASK_FLQ			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #define INSN_MATCH_FSW			0x2027
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #define INSN_MASK_FSW			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #define INSN_MATCH_FSD			0x3027
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #define INSN_MASK_FSD			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #define INSN_MATCH_FSQ			0x4027
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) #define INSN_MASK_FSQ			0x707f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #define INSN_MATCH_C_LD			0x6000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #define INSN_MASK_C_LD			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #define INSN_MATCH_C_SD			0xe000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #define INSN_MASK_C_SD			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #define INSN_MATCH_C_LW			0x4000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) #define INSN_MASK_C_LW			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) #define INSN_MATCH_C_SW			0xc000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #define INSN_MASK_C_SW			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) #define INSN_MATCH_C_LDSP		0x6002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) #define INSN_MASK_C_LDSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) #define INSN_MATCH_C_SDSP		0xe002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) #define INSN_MASK_C_SDSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) #define INSN_MATCH_C_LWSP		0x4002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define INSN_MASK_C_LWSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) #define INSN_MATCH_C_SWSP		0xc002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) #define INSN_MASK_C_SWSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) #define INSN_MATCH_C_FLD		0x2000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) #define INSN_MASK_C_FLD			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) #define INSN_MATCH_C_FLW		0x6000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) #define INSN_MASK_C_FLW			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #define INSN_MATCH_C_FSD		0xa000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) #define INSN_MASK_C_FSD			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) #define INSN_MATCH_C_FSW		0xe000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #define INSN_MASK_C_FSW			0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) #define INSN_MATCH_C_FLDSP		0x2002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) #define INSN_MASK_C_FLDSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) #define INSN_MATCH_C_FSDSP		0xa002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) #define INSN_MASK_C_FSDSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) #define INSN_MATCH_C_FLWSP		0x6002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) #define INSN_MASK_C_FLWSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) #define INSN_MATCH_C_FSWSP		0xe002
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) #define INSN_MASK_C_FSWSP		0xe003
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) #define INSN_LEN(insn)			((((insn) & 0x3) < 0x3) ? 2 : 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) #define LOG_REGBYTES			3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) #define XLEN				64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) #define LOG_REGBYTES			2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) #define XLEN				32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) #define REGBYTES			(1 << LOG_REGBYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) #define XLEN_MINUS_16			((XLEN) - 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) #define SH_RD				7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) #define SH_RS1				15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) #define SH_RS2				20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define SH_RS2C				2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define RV_X(x, s, n)			(((x) >> (s)) & ((1 << (n)) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define RVC_LW_IMM(x)			((RV_X(x, 6, 1) << 2) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 					 (RV_X(x, 10, 3) << 3) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 					 (RV_X(x, 5, 1) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define RVC_LD_IMM(x)			((RV_X(x, 10, 3) << 3) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 					 (RV_X(x, 5, 2) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #define RVC_LWSP_IMM(x)			((RV_X(x, 4, 3) << 2) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 					 (RV_X(x, 12, 1) << 5) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 					 (RV_X(x, 2, 2) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define RVC_LDSP_IMM(x)			((RV_X(x, 5, 2) << 3) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 					 (RV_X(x, 12, 1) << 5) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 					 (RV_X(x, 2, 3) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define RVC_SWSP_IMM(x)			((RV_X(x, 9, 4) << 2) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 					 (RV_X(x, 7, 2) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define RVC_SDSP_IMM(x)			((RV_X(x, 10, 3) << 3) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 					 (RV_X(x, 7, 3) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #define RVC_RS1S(insn)			(8 + RV_X(insn, SH_RD, 3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define RVC_RS2S(insn)			(8 + RV_X(insn, SH_RS2C, 3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define RVC_RS2(insn)			RV_X(insn, SH_RS2C, 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define SHIFT_RIGHT(x, y)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define REG_MASK			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define REG_OFFSET(insn, pos)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	(SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define REG_PTR(insn, pos, regs)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	(ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define GET_RM(insn)			(((insn) >> 12) & 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define GET_RS1(insn, regs)		(*REG_PTR(insn, SH_RS1, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define GET_RS2(insn, regs)		(*REG_PTR(insn, SH_RS2, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define GET_RS1S(insn, regs)		(*REG_PTR(RVC_RS1S(insn), 0, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define GET_RS2S(insn, regs)		(*REG_PTR(RVC_RS2S(insn), 0, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define GET_RS2C(insn, regs)		(*REG_PTR(insn, SH_RS2C, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define GET_SP(regs)			(*REG_PTR(2, 0, regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define SET_RD(insn, regs, val)		(*REG_PTR(insn, SH_RD, regs) = (val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define IMM_I(insn)			((s32)(insn) >> 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #define IMM_S(insn)			(((s32)(insn) >> 25 << 5) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 					 (s32)(((insn) >> 7) & 0x1f))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #define MASK_FUNCT3			0x7000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define GET_PRECISION(insn) (((insn) >> 25) & 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define GET_RM(insn) (((insn) >> 12) & 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define PRECISION_S 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define PRECISION_D 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define STR(x) XSTR(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define XSTR(x) #x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define DECLARE_UNPRIVILEGED_LOAD_FUNCTION(type, insn)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static inline type load_##type(const type *addr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	type val;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	asm (#insn " %0, %1"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	: "=&r" (val) : "m" (*addr));					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	return val;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define DECLARE_UNPRIVILEGED_STORE_FUNCTION(type, insn)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static inline void store_##type(type *addr, type val)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	asm volatile (#insn " %0, %1\n"					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	: : "r" (val), "m" (*addr));					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u8, lbu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u16, lhu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s8, lb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s16, lh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(s32, lw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) DECLARE_UNPRIVILEGED_STORE_FUNCTION(u8, sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) DECLARE_UNPRIVILEGED_STORE_FUNCTION(u16, sh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) DECLARE_UNPRIVILEGED_STORE_FUNCTION(u32, sw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lwu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u64, ld)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) DECLARE_UNPRIVILEGED_STORE_FUNCTION(u64, sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, ld)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(u32, lw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) DECLARE_UNPRIVILEGED_LOAD_FUNCTION(ulong, lw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static inline u64 load_u64(const u64 *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	return load_u32((u32 *)addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 		+ ((u64)load_u32((u32 *)addr + 1) << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) static inline void store_u64(u64 *addr, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	store_u32((u32 *)addr, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	store_u32((u32 *)addr + 1, val >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static inline ulong get_insn(ulong mepc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	register ulong __mepc asm ("a2") = mepc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	ulong val, rvc_mask = 3, tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	asm ("and %[tmp], %[addr], 2\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		"bnez %[tmp], 1f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		STR(LWU) " %[insn], (%[addr])\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		STR(LW) " %[insn], (%[addr])\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		"and %[tmp], %[insn], %[rvc_mask]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		"beq %[tmp], %[rvc_mask], 2f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 		"sll %[insn], %[insn], %[xlen_minus_16]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		"srl %[insn], %[insn], %[xlen_minus_16]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		"j 2f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		"1:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		"lhu %[insn], (%[addr])\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		"and %[tmp], %[insn], %[rvc_mask]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		"bne %[tmp], %[rvc_mask], 2f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		"lhu %[tmp], 2(%[addr])\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		"sll %[tmp], %[tmp], 16\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		"add %[insn], %[insn], %[tmp]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		"2:"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	: [insn] "=&r" (val), [tmp] "=&r" (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	: [addr] "r" (__mepc), [rvc_mask] "r" (rvc_mask),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	  [xlen_minus_16] "i" (XLEN_MINUS_16));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) union reg_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	u8 data_bytes[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	ulong data_ulong;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	u64 data_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) int handle_misaligned_load(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	union reg_data val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	unsigned long epc = regs->epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	unsigned long insn = get_insn(epc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	unsigned long addr = csr_read(mtval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	int i, fp = 0, shift = 0, len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	regs->epc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	} else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	} else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	} else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	} else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	} else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		len = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	} else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		len = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	} else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		insn = RVC_RS2S(insn) << SH_RD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	} else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 		   ((insn >> SH_RD) & 0x1f)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	} else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 		insn = RVC_RS2S(insn) << SH_RD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	} else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 		   ((insn >> SH_RD) & 0x1f)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 		shift = 8 * (sizeof(unsigned long) - len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	} else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		insn = RVC_RS2S(insn) << SH_RD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	} else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #if defined(CONFIG_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	} else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		insn = RVC_RS2S(insn) << SH_RD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	} else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		fp = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 		regs->epc = epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 		return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	val.data_u64 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	for (i = 0; i < len; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 		val.data_bytes[i] = load_u8((void *)(addr + i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	if (fp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	SET_RD(insn, regs, val.data_ulong << shift >> shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	regs->epc = epc + INSN_LEN(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) int handle_misaligned_store(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	union reg_data val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	unsigned long epc = regs->epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	unsigned long insn = get_insn(epc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	unsigned long addr = csr_read(mtval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	int i, len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	regs->epc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	val.data_ulong = GET_RS2(insn, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	} else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	} else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 		len = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #if defined(CONFIG_64BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	} else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		val.data_ulong = GET_RS2S(insn, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	} else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		   ((insn >> SH_RD) & 0x1f)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		len = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 		val.data_ulong = GET_RS2C(insn, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	} else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 		val.data_ulong = GET_RS2S(insn, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	} else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		   ((insn >> SH_RD) & 0x1f)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 		len = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 		val.data_ulong = GET_RS2C(insn, regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 		regs->epc = epc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 		return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	for (i = 0; i < len; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 		store_u8((void *)(addr + i), val.data_bytes[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	regs->epc = epc + INSN_LEN(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }