Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #define AES_FUNC_START(func)		SYM_FUNC_START(neon_ ## func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #define AES_FUNC_END(func)		SYM_FUNC_END(neon_ ## func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 	xtsmask		.req	v7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 	cbciv		.req	v7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	vctr		.req	v4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	.macro		xts_reload_mask, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	xts_load_mask	\tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	/* special case for the neon-bs driver calling into this one for CTS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	.macro		xts_cts_skip_tw, reg, lbl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	tbnz		\reg, #1, \lbl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	/* multiply by polynomial 'x' in GF(2^8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	.macro		mul_by_x, out, in, temp, const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	sshr		\temp, \in, #7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	shl		\out, \in, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	and		\temp, \temp, \const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	eor		\out, \out, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	/* multiply by polynomial 'x^2' in GF(2^8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	.macro		mul_by_x2, out, in, temp, const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	ushr		\temp, \in, #6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	shl		\out, \in, #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	pmul		\temp, \temp, \const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	eor		\out, \out, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	/* preload the entire Sbox */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	.macro		prepare, sbox, shiftrows, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	movi		v12.16b, #0x1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	ldr_l		q13, \shiftrows, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	ldr_l		q14, .Lror32by8, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	adr_l		\temp, \sbox
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	ld1		{v16.16b-v19.16b}, [\temp], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	ld1		{v20.16b-v23.16b}, [\temp], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	ld1		{v24.16b-v27.16b}, [\temp], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	ld1		{v28.16b-v31.16b}, [\temp]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	/* do preload for encryption */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	.macro		enc_prepare, ignore0, ignore1, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	prepare		crypto_aes_sbox, .LForward_ShiftRows, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	.macro		enc_switch_key, ignore0, ignore1, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	/* do nothing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	/* do preload for decryption */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	.macro		dec_prepare, ignore0, ignore1, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	prepare		crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	/* apply SubBytes transformation using the the preloaded Sbox */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	.macro		sub_bytes, in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	sub		v9.16b, \in\().16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	tbl		\in\().16b, {v16.16b-v19.16b}, \in\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	sub		v10.16b, v9.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	tbx		\in\().16b, {v20.16b-v23.16b}, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	sub		v11.16b, v10.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	tbx		\in\().16b, {v24.16b-v27.16b}, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	tbx		\in\().16b, {v28.16b-v31.16b}, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	/* apply MixColumns transformation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	.macro		mix_columns, in, enc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	.if		\enc == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	mul_by_x2	v8.16b, \in\().16b, v9.16b, v12.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	eor		\in\().16b, \in\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	rev32		v8.8h, v8.8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	eor		\in\().16b, \in\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	mul_by_x	v9.16b, \in\().16b, v8.16b, v12.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	rev32		v8.8h, \in\().8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	eor		v8.16b, v8.16b, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	eor		\in\().16b, \in\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	tbl		\in\().16b, {\in\().16b}, v14.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	eor		\in\().16b, \in\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	.macro		do_block, enc, in, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	ld1		{v15.4s}, [\rk]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	add		\rkp, \rk, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	mov		\i, \rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 1111:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	movi		v15.16b, #0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	tbl		\in\().16b, {\in\().16b}, v13.16b	/* ShiftRows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	sub_bytes	\in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	subs		\i, \i, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	ld1		{v15.4s}, [\rkp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	beq		2222f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	mix_columns	\in, \enc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	b		1111b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 2222:	eor		\in\().16b, \in\().16b, v15.16b		/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	.macro		encrypt_block, in, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	do_block	1, \in, \rounds, \rk, \rkp, \i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	.macro		decrypt_block, in, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	do_block	0, \in, \rounds, \rk, \rkp, \i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	 * Interleaved versions: functionally equivalent to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	 * ones above, but applied to AES states in parallel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	.macro		sub_bytes_4x, in0, in1, in2, in3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	sub		v8.16b, \in0\().16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	tbl		\in0\().16b, {v16.16b-v19.16b}, \in0\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	sub		v9.16b, \in1\().16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	tbl		\in1\().16b, {v16.16b-v19.16b}, \in1\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	sub		v10.16b, \in2\().16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	tbl		\in2\().16b, {v16.16b-v19.16b}, \in2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	sub		v11.16b, \in3\().16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	tbl		\in3\().16b, {v16.16b-v19.16b}, \in3\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	tbx		\in0\().16b, {v20.16b-v23.16b}, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	tbx		\in1\().16b, {v20.16b-v23.16b}, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	sub		v8.16b, v8.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	tbx		\in2\().16b, {v20.16b-v23.16b}, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	sub		v9.16b, v9.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	tbx		\in3\().16b, {v20.16b-v23.16b}, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	sub		v10.16b, v10.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	tbx		\in0\().16b, {v24.16b-v27.16b}, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	sub		v11.16b, v11.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	tbx		\in1\().16b, {v24.16b-v27.16b}, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	sub		v8.16b, v8.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	tbx		\in2\().16b, {v24.16b-v27.16b}, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	sub		v9.16b, v9.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	tbx		\in3\().16b, {v24.16b-v27.16b}, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	sub		v10.16b, v10.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	tbx		\in0\().16b, {v28.16b-v31.16b}, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	sub		v11.16b, v11.16b, v15.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	tbx		\in1\().16b, {v28.16b-v31.16b}, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	tbx		\in2\().16b, {v28.16b-v31.16b}, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	tbx		\in3\().16b, {v28.16b-v31.16b}, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	.macro		mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	sshr		\tmp0\().16b, \in0\().16b, #7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	shl		\out0\().16b, \in0\().16b, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	sshr		\tmp1\().16b, \in1\().16b, #7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	and		\tmp0\().16b, \tmp0\().16b, \const\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	shl		\out1\().16b, \in1\().16b, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	and		\tmp1\().16b, \tmp1\().16b, \const\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	.macro		mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	ushr		\tmp0\().16b, \in0\().16b, #6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	shl		\out0\().16b, \in0\().16b, #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	ushr		\tmp1\().16b, \in1\().16b, #6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	pmul		\tmp0\().16b, \tmp0\().16b, \const\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	shl		\out1\().16b, \in1\().16b, #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	pmul		\tmp1\().16b, \tmp1\().16b, \const\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	eor		\out0\().16b, \out0\().16b, \tmp0\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	eor		\out1\().16b, \out1\().16b, \tmp1\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	.macro		mix_columns_2x, in0, in1, enc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	.if		\enc == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	mul_by_x2_2x	v8, v9, \in0, \in1, v10, v11, v12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	eor		\in0\().16b, \in0\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	rev32		v8.8h, v8.8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	eor		\in1\().16b, \in1\().16b, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	rev32		v9.8h, v9.8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	eor		\in0\().16b, \in0\().16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	eor		\in1\().16b, \in1\().16b, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	mul_by_x_2x	v8, v9, \in0, \in1, v10, v11, v12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	rev32		v10.8h, \in0\().8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	rev32		v11.8h, \in1\().8h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	eor		v10.16b, v10.16b, v8.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	eor		v11.16b, v11.16b, v9.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	eor		\in0\().16b, \in0\().16b, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	eor		\in1\().16b, \in1\().16b, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	tbl		\in0\().16b, {\in0\().16b}, v14.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	tbl		\in1\().16b, {\in1\().16b}, v14.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	eor		\in0\().16b, \in0\().16b, v10.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	eor		\in1\().16b, \in1\().16b, v11.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	.macro		do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	ld1		{v15.4s}, [\rk]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	add		\rkp, \rk, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	mov		\i, \rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 1111:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	movi		v15.16b, #0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	tbl		\in0\().16b, {\in0\().16b}, v13.16b	/* ShiftRows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	tbl		\in1\().16b, {\in1\().16b}, v13.16b	/* ShiftRows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	tbl		\in2\().16b, {\in2\().16b}, v13.16b	/* ShiftRows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	tbl		\in3\().16b, {\in3\().16b}, v13.16b	/* ShiftRows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	sub_bytes_4x	\in0, \in1, \in2, \in3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	subs		\i, \i, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	ld1		{v15.4s}, [\rkp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	beq		2222f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	mix_columns_2x	\in0, \in1, \enc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	mix_columns_2x	\in2, \in3, \enc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	b		1111b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 2222:	eor		\in0\().16b, \in0\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	eor		\in1\().16b, \in1\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	eor		\in2\().16b, \in2\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	eor		\in3\().16b, \in3\().16b, v15.16b	/* ^round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	.macro		encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	do_block_4x	1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	.macro		decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	do_block_4x	0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) #include "aes-modes.S"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	.section	".rodata", "a"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	.align		4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) .LForward_ShiftRows:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	.octa		0x0b06010c07020d08030e09040f0a0500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) .LReverse_ShiftRows:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	.octa		0x0306090c0f0205080b0e0104070a0d00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) .Lror32by8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	.octa		0x0c0f0e0d080b0a090407060500030201