Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * FP/SIMD state saving and restoring macros
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2012 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Author: Catalin Marinas <catalin.marinas@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) .macro fpsimd_save state, tmpnr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 	stp	q0, q1, [\state, #16 * 0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 	stp	q2, q3, [\state, #16 * 2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 	stp	q4, q5, [\state, #16 * 4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 	stp	q6, q7, [\state, #16 * 6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	stp	q8, q9, [\state, #16 * 8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	stp	q10, q11, [\state, #16 * 10]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	stp	q12, q13, [\state, #16 * 12]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	stp	q14, q15, [\state, #16 * 14]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	stp	q16, q17, [\state, #16 * 16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	stp	q18, q19, [\state, #16 * 18]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	stp	q20, q21, [\state, #16 * 20]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	stp	q22, q23, [\state, #16 * 22]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	stp	q24, q25, [\state, #16 * 24]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	stp	q26, q27, [\state, #16 * 26]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	stp	q28, q29, [\state, #16 * 28]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	stp	q30, q31, [\state, #16 * 30]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	mrs	x\tmpnr, fpsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	str	w\tmpnr, [\state, #16 * 2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	mrs	x\tmpnr, fpcr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	str	w\tmpnr, [\state, #16 * 2 + 4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) .macro fpsimd_restore_fpcr state, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	 * Writes to fpcr may be self-synchronising, so avoid restoring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	 * the register if it hasn't changed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	mrs	\tmp, fpcr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	cmp	\tmp, \state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	b.eq	9999f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	msr	fpcr, \state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 9999:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) /* Clobbers \state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) .macro fpsimd_restore state, tmpnr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	ldp	q0, q1, [\state, #16 * 0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	ldp	q2, q3, [\state, #16 * 2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	ldp	q4, q5, [\state, #16 * 4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	ldp	q6, q7, [\state, #16 * 6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	ldp	q8, q9, [\state, #16 * 8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	ldp	q10, q11, [\state, #16 * 10]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	ldp	q12, q13, [\state, #16 * 12]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	ldp	q14, q15, [\state, #16 * 14]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	ldp	q16, q17, [\state, #16 * 16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	ldp	q18, q19, [\state, #16 * 18]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	ldp	q20, q21, [\state, #16 * 20]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	ldp	q22, q23, [\state, #16 * 22]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	ldp	q24, q25, [\state, #16 * 24]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	ldp	q26, q27, [\state, #16 * 26]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	ldp	q28, q29, [\state, #16 * 28]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	ldp	q30, q31, [\state, #16 * 30]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	ldr	w\tmpnr, [\state, #16 * 2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	msr	fpsr, x\tmpnr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	ldr	w\tmpnr, [\state, #16 * 2 + 4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	fpsimd_restore_fpcr x\tmpnr, \state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) /* Sanity-check macros to help avoid encoding garbage instructions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) .macro _check_general_reg nr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	.if (\nr) < 0 || (\nr) > 30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		.error "Bad register number \nr."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) .macro _sve_check_zreg znr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	.if (\znr) < 0 || (\znr) > 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		.error "Bad Scalable Vector Extension vector register number \znr."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) .macro _sve_check_preg pnr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	.if (\pnr) < 0 || (\pnr) > 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		.error "Bad Scalable Vector Extension predicate register number \pnr."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) .macro _check_num n, min, max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	.if (\n) < (\min) || (\n) > (\max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		.error "Number \n out of range [\min,\max]"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) /* SVE instruction encodings for non-SVE-capable assemblers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) .macro _sve_str_v nz, nxbase, offset=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	_sve_check_zreg \nz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	_check_general_reg \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	_check_num (\offset), -0x100, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	.inst	0xe5804000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		| (\nz)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		| ((\nxbase) << 5)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		| (((\offset) & 7) << 10)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		| (((\offset) & 0x1f8) << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .macro _sve_ldr_v nz, nxbase, offset=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	_sve_check_zreg \nz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	_check_general_reg \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	_check_num (\offset), -0x100, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	.inst	0x85804000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		| (\nz)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 		| ((\nxbase) << 5)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 		| (((\offset) & 7) << 10)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		| (((\offset) & 0x1f8) << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) .macro _sve_str_p np, nxbase, offset=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	_sve_check_preg \np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	_check_general_reg \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	_check_num (\offset), -0x100, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	.inst	0xe5800000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		| (\np)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		| ((\nxbase) << 5)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		| (((\offset) & 7) << 10)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		| (((\offset) & 0x1f8) << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) .macro _sve_ldr_p np, nxbase, offset=0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	_sve_check_preg \np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	_check_general_reg \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	_check_num (\offset), -0x100, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	.inst	0x85800000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 		| (\np)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 		| ((\nxbase) << 5)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		| (((\offset) & 7) << 10)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		| (((\offset) & 0x1f8) << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /* RDVL X\nx, #\imm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .macro _sve_rdvl nx, imm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	_check_general_reg \nx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	_check_num (\imm), -0x20, 0x1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	.inst	0x04bf5000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		| (\nx)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		| (((\imm) & 0x3f) << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* RDFFR (unpredicated): RDFFR P\np.B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .macro _sve_rdffr np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	_sve_check_preg \np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	.inst	0x2519f000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		| (\np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /* WRFFR P\np.B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .macro _sve_wrffr np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	_sve_check_preg \np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	.inst	0x25289000			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		| ((\np) << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /* PFALSE P\np.B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) .macro _sve_pfalse np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	_sve_check_preg \np
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	.inst	0x2518e400			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		| (\np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) .macro __for from:req, to:req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	.if (\from) == (\to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		_for__body %\from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	.else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		__for %\from, %((\from) + ((\to) - (\from)) / 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		__for %((\from) + ((\to) - (\from)) / 2 + 1), %\to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) .macro _for var:req, from:req, to:req, insn:vararg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	.macro _for__body \var:req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 		.noaltmacro
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		\insn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		.altmacro
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	.altmacro
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	__for \from, \to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	.noaltmacro
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	.purgem _for__body
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* Update ZCR_EL1.LEN with the new VQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) .macro sve_load_vq xvqminus1, xtmp, xtmp2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		mrs_s		\xtmp, SYS_ZCR_EL1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		bic		\xtmp2, \xtmp, ZCR_ELx_LEN_MASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		orr		\xtmp2, \xtmp2, \xvqminus1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		cmp		\xtmp2, \xtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 		b.eq		921f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 		msr_s		SYS_ZCR_EL1, \xtmp2	//self-synchronising
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 921:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* Preserve the first 128-bits of Znz and zero the rest. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) .macro _sve_flush_z nz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	_sve_check_zreg \nz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	mov	v\nz\().16b, v\nz\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) .macro sve_flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  _for n, 0, 31, _sve_flush_z	\n
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  _for n, 0, 15, _sve_pfalse	\n
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		_sve_wrffr	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) .macro sve_save nxbase, xpfpsr, nxtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)  _for n, 0, 31,	_sve_str_v	\n, \nxbase, \n - 34
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)  _for n, 0, 15,	_sve_str_p	\n, \nxbase, \n - 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		_sve_rdffr	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		_sve_str_p	0, \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		_sve_ldr_p	0, \nxbase, -16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		mrs		x\nxtmp, fpsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		str		w\nxtmp, [\xpfpsr]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		mrs		x\nxtmp, fpcr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 		str		w\nxtmp, [\xpfpsr, #4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) .macro __sve_load nxbase, xpfpsr, nxtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)  _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		_sve_ldr_p	0, \nxbase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 		_sve_wrffr	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)  _for n, 0, 15,	_sve_ldr_p	\n, \nxbase, \n - 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		ldr		w\nxtmp, [\xpfpsr]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		msr		fpsr, x\nxtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		ldr		w\nxtmp, [\xpfpsr, #4]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		msr		fpcr, x\nxtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) .macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 		sve_load_vq	\xvqminus1, x\nxtmp, \xtmp2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 		__sve_load	\nxbase, \xpfpsr, \nxtmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) .endm