^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /* included by aes-ce.S and aes-neon.S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) .text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) .align 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #ifndef MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define MAX_STRIDE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #if MAX_STRIDE == 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define ST4(x...) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define ST5(x...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define ST4(x...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define ST5(x...) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) SYM_FUNC_END(aes_encrypt_block4x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) SYM_FUNC_END(aes_decrypt_block4x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #if MAX_STRIDE == 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) SYM_FUNC_END(aes_encrypt_block5x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) SYM_FUNC_END(aes_decrypt_block5x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * int blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * int blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) AES_FUNC_START(aes_ecb_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) enc_prepare w3, x2, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) .LecbencloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) subs w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) bmi .Lecbenc1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) ST4( bl aes_encrypt_block4x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) ST5( ld1 {v4.16b}, [x1], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) ST5( bl aes_encrypt_block5x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ST5( st1 {v4.16b}, [x0], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) b .LecbencloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) .Lecbenc1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) adds w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) beq .Lecbencout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) .Lecbencloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ld1 {v0.16b}, [x1], #16 /* get next pt block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) encrypt_block v0, w3, x2, x5, w6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) st1 {v0.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) subs w4, w4, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) bne .Lecbencloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) .Lecbencout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) AES_FUNC_END(aes_ecb_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) AES_FUNC_START(aes_ecb_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) dec_prepare w3, x2, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .LecbdecloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) subs w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) bmi .Lecbdec1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ST4( bl aes_decrypt_block4x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ST5( ld1 {v4.16b}, [x1], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) ST5( bl aes_decrypt_block5x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ST5( st1 {v4.16b}, [x0], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) b .LecbdecloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) .Lecbdec1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) adds w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) beq .Lecbdecout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .Lecbdecloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) ld1 {v0.16b}, [x1], #16 /* get next ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) decrypt_block v0, w3, x2, x5, w6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) st1 {v0.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) subs w4, w4, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) bne .Lecbdecloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) .Lecbdecout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) AES_FUNC_END(aes_ecb_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * int blocks, u8 iv[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * int blocks, u8 iv[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * int rounds, int blocks, u8 iv[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * u32 const rk2[]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * int rounds, int blocks, u8 iv[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * u32 const rk2[]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) AES_FUNC_START(aes_essiv_cbc_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ld1 {v4.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) mov w8, #14 /* AES-256: 14 rounds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) enc_prepare w8, x6, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) encrypt_block v4, w8, x6, x7, w9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) enc_switch_key w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) b .Lcbcencloop4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) AES_FUNC_START(aes_cbc_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) ld1 {v4.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) enc_prepare w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .Lcbcencloop4x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) subs w4, w4, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) bmi .Lcbcenc1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) encrypt_block v0, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) eor v1.16b, v1.16b, v0.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) encrypt_block v1, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) eor v2.16b, v2.16b, v1.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) encrypt_block v2, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) eor v3.16b, v3.16b, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) encrypt_block v3, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) mov v4.16b, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) b .Lcbcencloop4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .Lcbcenc1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) adds w4, w4, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) beq .Lcbcencout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .Lcbcencloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) ld1 {v0.16b}, [x1], #16 /* get next pt block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) encrypt_block v4, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) st1 {v4.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) subs w4, w4, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) bne .Lcbcencloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .Lcbcencout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) st1 {v4.16b}, [x5] /* return iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) AES_FUNC_END(aes_cbc_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) AES_FUNC_END(aes_essiv_cbc_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) AES_FUNC_START(aes_essiv_cbc_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) ld1 {cbciv.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) mov w8, #14 /* AES-256: 14 rounds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) enc_prepare w8, x6, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) encrypt_block cbciv, w8, x6, x7, w9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) b .Lessivcbcdecstart
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) AES_FUNC_START(aes_cbc_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) ld1 {cbciv.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .Lessivcbcdecstart:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) dec_prepare w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) .LcbcdecloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) subs w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) bmi .Lcbcdec1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #if MAX_STRIDE == 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ld1 {v4.16b}, [x1], #16 /* get 1 ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mov v5.16b, v0.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) mov v6.16b, v1.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) mov v7.16b, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) bl aes_decrypt_block5x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) sub x1, x1, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) eor v0.16b, v0.16b, cbciv.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) eor v1.16b, v1.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) eor v2.16b, v2.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) eor v3.16b, v3.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) eor v4.16b, v4.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) mov v4.16b, v0.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) mov v5.16b, v1.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) mov v6.16b, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) bl aes_decrypt_block4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) sub x1, x1, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) eor v0.16b, v0.16b, cbciv.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) eor v1.16b, v1.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) eor v2.16b, v2.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) eor v3.16b, v3.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) ST5( st1 {v4.16b}, [x0], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) b .LcbcdecloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) .Lcbcdec1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) adds w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) beq .Lcbcdecout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) .Lcbcdecloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) ld1 {v1.16b}, [x1], #16 /* get next ct block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) mov v0.16b, v1.16b /* ...and copy to v0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) decrypt_block v0, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) mov cbciv.16b, v1.16b /* ct is next iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) st1 {v0.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) subs w4, w4, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) bne .Lcbcdecloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) .Lcbcdecout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) st1 {cbciv.16b}, [x5] /* return iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) AES_FUNC_END(aes_cbc_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) AES_FUNC_END(aes_essiv_cbc_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * int rounds, int bytes, u8 const iv[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * int rounds, int bytes, u8 const iv[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) AES_FUNC_START(aes_cbc_cts_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) adr_l x8, .Lcts_permute_table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) sub x4, x4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) add x9, x8, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) add x8, x8, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) sub x9, x9, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) ld1 {v3.16b}, [x8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) ld1 {v4.16b}, [x9]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) ld1 {v0.16b}, [x1], x4 /* overlapping loads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) ld1 {v1.16b}, [x1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) ld1 {v5.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) enc_prepare w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) eor v0.16b, v0.16b, v5.16b /* xor with iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) tbl v1.16b, {v1.16b}, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) encrypt_block v0, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) eor v1.16b, v1.16b, v0.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) tbl v0.16b, {v0.16b}, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) encrypt_block v1, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) add x4, x0, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) st1 {v0.16b}, [x4] /* overlapping stores */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) st1 {v1.16b}, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) AES_FUNC_END(aes_cbc_cts_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) AES_FUNC_START(aes_cbc_cts_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) adr_l x8, .Lcts_permute_table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) sub x4, x4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) add x9, x8, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) add x8, x8, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) sub x9, x9, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) ld1 {v3.16b}, [x8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) ld1 {v4.16b}, [x9]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ld1 {v0.16b}, [x1], x4 /* overlapping loads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) ld1 {v1.16b}, [x1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) ld1 {v5.16b}, [x5] /* get iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) dec_prepare w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) decrypt_block v0, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) tbl v2.16b, {v0.16b}, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) eor v2.16b, v2.16b, v1.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) tbx v0.16b, {v1.16b}, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) decrypt_block v0, w3, x2, x6, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) eor v0.16b, v0.16b, v5.16b /* xor with iv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) add x4, x0, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) st1 {v2.16b}, [x4] /* overlapping stores */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) st1 {v0.16b}, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) AES_FUNC_END(aes_cbc_cts_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) .section ".rodata", "a"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) .align 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) .Lcts_permute_table:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * int blocks, u8 ctr[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) AES_FUNC_START(aes_ctr_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) enc_prepare w3, x2, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ld1 {vctr.16b}, [x5]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) umov x6, vctr.d[1] /* keep swabbed ctr in reg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) rev x6, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) cmn w6, w4 /* 32 bit overflow? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) bcs .Lctrloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) .LctrloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) subs w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) bmi .Lctr1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) add w7, w6, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) mov v0.16b, vctr.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) add w8, w6, #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) mov v1.16b, vctr.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) add w9, w6, #3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) mov v2.16b, vctr.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) add w9, w6, #3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) rev w7, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) mov v3.16b, vctr.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) rev w8, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ST5( mov v4.16b, vctr.16b )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) mov v1.s[3], w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) rev w9, w9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) ST5( add w10, w6, #4 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) mov v2.s[3], w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) ST5( rev w10, w10 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) mov v3.s[3], w9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) ST5( mov v4.s[3], w10 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ST4( bl aes_encrypt_block4x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) ST5( bl aes_encrypt_block5x )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) eor v0.16b, v5.16b, v0.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) ST4( ld1 {v5.16b}, [x1], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) eor v1.16b, v6.16b, v1.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) ST5( ld1 {v5.16b-v6.16b}, [x1], #32 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) eor v2.16b, v7.16b, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) eor v3.16b, v5.16b, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) ST5( eor v4.16b, v6.16b, v4.16b )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) ST5( st1 {v4.16b}, [x0], #16 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) add x6, x6, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) rev x7, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) ins vctr.d[1], x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) cbz w4, .Lctrout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) b .LctrloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) .Lctr1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) adds w4, w4, #MAX_STRIDE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) beq .Lctrout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) .Lctrloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) mov v0.16b, vctr.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) encrypt_block v0, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) adds x6, x6, #1 /* increment BE ctr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) rev x7, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) ins vctr.d[1], x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) bcs .Lctrcarry /* overflow? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) .Lctrcarrydone:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) subs w4, w4, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) bmi .Lctrtailblock /* blocks <0 means tail block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) ld1 {v3.16b}, [x1], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) eor v3.16b, v0.16b, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) st1 {v3.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) bne .Lctrloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) .Lctrout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) st1 {vctr.16b}, [x5] /* return next CTR value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) .Lctrtailblock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) st1 {v0.16b}, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) b .Lctrout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) .Lctrcarry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) umov x7, vctr.d[0] /* load upper word of ctr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) rev x7, x7 /* ... to handle the carry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) add x7, x7, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) rev x7, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ins vctr.d[0], x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) b .Lctrcarrydone
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) AES_FUNC_END(aes_ctr_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * int bytes, u8 const rk2[], u8 iv[], int first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * int bytes, u8 const rk2[], u8 iv[], int first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) .macro next_tweak, out, in, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) sshr \tmp\().2d, \in\().2d, #63
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) and \tmp\().16b, \tmp\().16b, xtsmask.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) add \out\().2d, \in\().2d, \in\().2d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) eor \out\().16b, \out\().16b, \tmp\().16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) .macro xts_load_mask, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) movi xtsmask.2s, #0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) movi \tmp\().2s, #0x87
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) AES_FUNC_START(aes_xts_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) ld1 {v4.16b}, [x6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) xts_load_mask v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) cbz w7, .Lxtsencnotfirst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) enc_prepare w3, x5, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) xts_cts_skip_tw w7, .LxtsencNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) encrypt_block v4, w3, x5, x8, w7 /* first tweak */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) enc_switch_key w3, x2, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) b .LxtsencNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .Lxtsencnotfirst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) enc_prepare w3, x2, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) .LxtsencloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) next_tweak v4, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) .LxtsencNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) subs w4, w4, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) bmi .Lxtsenc1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) next_tweak v5, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) next_tweak v6, v5, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) eor v1.16b, v1.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) eor v2.16b, v2.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) next_tweak v7, v6, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) eor v3.16b, v3.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) bl aes_encrypt_block4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) eor v3.16b, v3.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) eor v1.16b, v1.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) eor v2.16b, v2.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) mov v4.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) cbz w4, .Lxtsencret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) xts_reload_mask v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) b .LxtsencloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) .Lxtsenc1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) adds w4, w4, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) beq .Lxtsencout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) subs w4, w4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) bmi .LxtsencctsNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) .Lxtsencloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) ld1 {v0.16b}, [x1], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) .Lxtsencctsout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) encrypt_block v0, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) cbz w4, .Lxtsencout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) subs w4, w4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) next_tweak v4, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) bmi .Lxtsenccts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) st1 {v0.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) b .Lxtsencloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) .Lxtsencout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) st1 {v0.16b}, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) .Lxtsencret:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) st1 {v4.16b}, [x6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) .LxtsencctsNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) mov v0.16b, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) sub x0, x0, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) .Lxtsenccts:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) adr_l x8, .Lcts_permute_table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) add x1, x1, w4, sxtw /* rewind input pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) add w4, w4, #16 /* # bytes in final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) add x9, x8, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) add x8, x8, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) sub x9, x9, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) add x4, x0, x4 /* output address of final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ld1 {v1.16b}, [x1] /* load final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) ld1 {v2.16b}, [x8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) ld1 {v3.16b}, [x9]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) tbl v2.16b, {v0.16b}, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) tbx v0.16b, {v1.16b}, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) st1 {v2.16b}, [x4] /* overlapping stores */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) mov w4, wzr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) b .Lxtsencctsout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) AES_FUNC_END(aes_xts_encrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) AES_FUNC_START(aes_xts_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) stp x29, x30, [sp, #-16]!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) mov x29, sp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* subtract 16 bytes if we are doing CTS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) sub w8, w4, #0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) tst w4, #0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) csel w4, w4, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) ld1 {v4.16b}, [x6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) xts_load_mask v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) xts_cts_skip_tw w7, .Lxtsdecskiptw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) cbz w7, .Lxtsdecnotfirst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) enc_prepare w3, x5, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) encrypt_block v4, w3, x5, x8, w7 /* first tweak */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) .Lxtsdecskiptw:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) dec_prepare w3, x2, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) b .LxtsdecNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) .Lxtsdecnotfirst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) dec_prepare w3, x2, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) .LxtsdecloopNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) next_tweak v4, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) .LxtsdecNx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) subs w4, w4, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) bmi .Lxtsdec1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) next_tweak v5, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) next_tweak v6, v5, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) eor v1.16b, v1.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) eor v2.16b, v2.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) next_tweak v7, v6, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) eor v3.16b, v3.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) bl aes_decrypt_block4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) eor v3.16b, v3.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) eor v1.16b, v1.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) eor v2.16b, v2.16b, v6.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) st1 {v0.16b-v3.16b}, [x0], #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) mov v4.16b, v7.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) cbz w4, .Lxtsdecout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) xts_reload_mask v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) b .LxtsdecloopNx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) .Lxtsdec1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) adds w4, w4, #64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) beq .Lxtsdecout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) subs w4, w4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) .Lxtsdecloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) ld1 {v0.16b}, [x1], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) bmi .Lxtsdeccts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) .Lxtsdecctsout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) decrypt_block v0, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) st1 {v0.16b}, [x0], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) cbz w4, .Lxtsdecout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) subs w4, w4, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) next_tweak v4, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) b .Lxtsdecloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) .Lxtsdecout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) st1 {v4.16b}, [x6]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) ldp x29, x30, [sp], #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .Lxtsdeccts:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) adr_l x8, .Lcts_permute_table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) add x1, x1, w4, sxtw /* rewind input pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) add w4, w4, #16 /* # bytes in final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) add x9, x8, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) add x8, x8, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) sub x9, x9, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) add x4, x0, x4 /* output address of final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) next_tweak v5, v4, v8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) ld1 {v1.16b}, [x1] /* load final block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) ld1 {v2.16b}, [x8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ld1 {v3.16b}, [x9]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) eor v0.16b, v0.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) decrypt_block v0, w3, x2, x8, w7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) eor v0.16b, v0.16b, v5.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) tbl v2.16b, {v0.16b}, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) tbx v0.16b, {v1.16b}, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) st1 {v2.16b}, [x4] /* overlapping stores */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) mov w4, wzr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) b .Lxtsdecctsout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) AES_FUNC_END(aes_xts_decrypt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * aes_mac_update(u8 const in[], u32 const rk[], int rounds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * int blocks, u8 dg[], int enc_before, int enc_after)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) AES_FUNC_START(aes_mac_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) ld1 {v0.16b}, [x4] /* get dg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) enc_prepare w2, x1, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) cbz w5, .Lmacloop4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) .Lmacloop4x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) subs w3, w3, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) bmi .Lmac1x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) eor v0.16b, v0.16b, v2.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) eor v0.16b, v0.16b, v3.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) eor v0.16b, v0.16b, v4.16b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) cmp w3, wzr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) csinv x5, x6, xzr, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) cbz w5, .Lmacout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) st1 {v0.16b}, [x4] /* return dg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) cond_yield .Lmacout, x7, x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) b .Lmacloop4x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) .Lmac1x:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) add w3, w3, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) .Lmacloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) cbz w3, .Lmacout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) ld1 {v1.16b}, [x0], #16 /* get next pt block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) subs w3, w3, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) csinv x5, x6, xzr, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) cbz w5, .Lmacout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) .Lmacenc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) encrypt_block v0, w2, x1, x7, w8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) b .Lmacloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) .Lmacout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) st1 {v0.16b}, [x4] /* return dg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) mov w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) AES_FUNC_END(aes_mac_update)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags