Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *                                    Crypto Extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #define AES_FUNC_START(func)		SYM_FUNC_START(ce_ ## func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #define AES_FUNC_END(func)		SYM_FUNC_END(ce_ ## func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 	.arch		armv8-a+crypto
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	xtsmask		.req	v16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	cbciv		.req	v16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	vctr		.req	v16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	.macro		xts_reload_mask, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	.macro		xts_cts_skip_tw, reg, lbl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	/* preload all round keys */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	.macro		load_round_keys, rounds, rk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	cmp		\rounds, #12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	blo		2222f		/* 128 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	beq		1111f		/* 192 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	ld1		{v17.4s-v18.4s}, [\rk], #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 1111:	ld1		{v19.4s-v20.4s}, [\rk], #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 2222:	ld1		{v21.4s-v24.4s}, [\rk], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	ld1		{v25.4s-v28.4s}, [\rk], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	ld1		{v29.4s-v31.4s}, [\rk]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	/* prepare for encryption with key in rk[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	.macro		enc_prepare, rounds, rk, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	mov		\temp, \rk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	load_round_keys	\rounds, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	/* prepare for encryption (again) but with new key in rk[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	.macro		enc_switch_key, rounds, rk, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	mov		\temp, \rk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	load_round_keys	\rounds, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	/* prepare for decryption with key in rk[] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	.macro		dec_prepare, rounds, rk, temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	mov		\temp, \rk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	load_round_keys	\rounds, \temp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	.macro		do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	aes\de		\i0\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	aes\mc		\i0\().16b, \i0\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	.ifnb		\i1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	aes\de		\i1\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	aes\mc		\i1\().16b, \i1\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	.ifnb		\i3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	aes\de		\i2\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	aes\mc		\i2\().16b, \i2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	aes\de		\i3\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	aes\mc		\i3\().16b, \i3\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	.ifnb		\i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	aes\de		\i4\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	aes\mc		\i4\().16b, \i4\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	/* up to 5 interleaved encryption rounds with the same round key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	.macro		round_Nx, enc, k, i0, i1, i2, i3, i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	.ifc		\enc, e
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	do_enc_Nx	e, mc, \k, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	.else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	do_enc_Nx	d, imc, \k, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	/* up to 5 interleaved final rounds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	.macro		fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	aes\de		\i0\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	.ifnb		\i1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	aes\de		\i1\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	.ifnb		\i3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	aes\de		\i2\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	aes\de		\i3\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	.ifnb		\i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	aes\de		\i4\().16b, \k\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	eor		\i0\().16b, \i0\().16b, \k2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	.ifnb		\i1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	eor		\i1\().16b, \i1\().16b, \k2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	.ifnb		\i3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	eor		\i2\().16b, \i2\().16b, \k2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	eor		\i3\().16b, \i3\().16b, \k2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	.ifnb		\i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	eor		\i4\().16b, \i4\().16b, \k2\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	/* up to 5 interleaved blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	.macro		do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	cmp		\rounds, #12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	blo		2222f		/* 128 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	beq		1111f		/* 192 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	round_Nx	\enc, v17, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	round_Nx	\enc, v18, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 1111:	round_Nx	\enc, v19, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	round_Nx	\enc, v20, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 2222:	.irp		key, v21, v22, v23, v24, v25, v26, v27, v28, v29
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	round_Nx	\enc, \key, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	.endr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	fin_round_Nx	\enc, v30, v31, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	.macro		encrypt_block, in, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	do_block_Nx	e, \rounds, \in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	.macro		encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	.macro		encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	do_block_Nx	e, \rounds, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	.macro		decrypt_block, in, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	do_block_Nx	d, \rounds, \in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	.macro		decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	.macro		decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	do_block_Nx	d, \rounds, \i0, \i1, \i2, \i3, \i4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define MAX_STRIDE	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #include "aes-modes.S"