Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * AES-NI + SSE2 implementation of AEGIS-128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/frame.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #define STATE0	%xmm0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #define STATE1	%xmm1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #define STATE2	%xmm2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #define STATE3	%xmm3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #define STATE4	%xmm4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define KEY	%xmm5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #define MSG	%xmm5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define T0	%xmm6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define T1	%xmm7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define STATEP	%rdi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #define LEN	%rsi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define SRC	%rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define DST	%rcx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) .section .rodata.cst16.aegis128_const, "aM", @progbits, 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) .Laegis128_const_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	.byte 0x00, 0x01, 0x01, 0x02, 0x03, 0x05, 0x08, 0x0d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	.byte 0x15, 0x22, 0x37, 0x59, 0x90, 0xe9, 0x79, 0x62
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) .Laegis128_const_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	.byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	.byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) .section .rodata.cst16.aegis128_counter, "aM", @progbits, 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) .Laegis128_counter:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) .text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * input:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  *   STATE[0-4] - input state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  * output:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  *   STATE[0-4] - output state (shifted positions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  * changed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  *   T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) .macro aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	movdqa STATE4, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	aesenc STATE0, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	aesenc STATE1, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	aesenc STATE2, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	aesenc STATE3, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	aesenc T0,     STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63)  * __load_partial: internal ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64)  * input:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65)  *   LEN - bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66)  *   SRC - src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67)  * output:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68)  *   MSG  - message block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69)  * changed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  *   T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  *   %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72)  *   %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) SYM_FUNC_START_LOCAL(__load_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	xor %r9d, %r9d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	pxor MSG, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	and $0x1, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	jz .Lld_partial_1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	and $0x1E, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	add SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	mov (%r8), %r9b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) .Lld_partial_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	and $0x2, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	jz .Lld_partial_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	and $0x1C, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	add SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	shl $0x10, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	mov (%r8), %r9w
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) .Lld_partial_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	and $0x4, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	jz .Lld_partial_4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	and $0x18, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	add SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	shl $32, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	mov (%r8), %r8d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	xor %r8, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) .Lld_partial_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	movq %r9, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	and $0x8, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	jz .Lld_partial_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	and $0x10, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	add SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	pslldq $8, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	movq (%r8), T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	pxor T0, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) .Lld_partial_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) SYM_FUNC_END(__load_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)  * __store_partial: internal ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)  * input:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)  *   LEN - bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)  *   DST - dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)  * output:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)  *   T0   - message block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)  * changed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)  *   %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)  *   %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)  *   %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) SYM_FUNC_START_LOCAL(__store_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	mov LEN, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	mov DST, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	movq T0, %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	cmp $8, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	jl .Lst_partial_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	mov %r10, (%r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	psrldq $8, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	movq T0, %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	sub $8, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	add $8, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .Lst_partial_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	cmp $4, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	jl .Lst_partial_4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	mov %r10d, (%r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	shr $32, %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	sub $4, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	add $4, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .Lst_partial_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	cmp $2, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	jl .Lst_partial_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	mov %r10w, (%r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	shr $0x10, %r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	sub $2, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	add $2, %r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) .Lst_partial_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	cmp $1, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	jl .Lst_partial_1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	mov %r10b, (%r9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) .Lst_partial_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) SYM_FUNC_END(__store_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)  * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) SYM_FUNC_START(crypto_aegis128_aesni_init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	/* load IV: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	movdqu (%rdx), T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	/* load key: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	movdqa (%rsi), KEY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	pxor KEY, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	movdqa T1, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	movdqa KEY, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	movdqa KEY, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	/* load the constants: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	movdqa .Laegis128_const_0, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	movdqa .Laegis128_const_1, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	pxor STATE2, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	pxor STATE1, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	/* update 10 times with KEY / KEY xor IV: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	aegis128_update; pxor KEY, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	aegis128_update; pxor T1,  STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	aegis128_update; pxor KEY, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	aegis128_update; pxor T1,  STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	aegis128_update; pxor KEY, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	aegis128_update; pxor T1,  STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	aegis128_update; pxor KEY, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	aegis128_update; pxor T1,  STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	aegis128_update; pxor KEY, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	aegis128_update; pxor T1,  STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	movdqu STATE0, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	movdqu STATE1, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	movdqu STATE2, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	movdqu STATE3, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	movdqu STATE4, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) SYM_FUNC_END(crypto_aegis128_aesni_init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)  * void crypto_aegis128_aesni_ad(void *state, unsigned int length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)  *                               const void *data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) SYM_FUNC_START(crypto_aegis128_aesni_ad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	jb .Lad_out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	mov SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	and $0xF, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	jnz .Lad_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) .Lad_a_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	movdqa 0x00(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	jl .Lad_out_1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	movdqa 0x10(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	pxor MSG, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	jl .Lad_out_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	movdqa 0x20(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	pxor MSG, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	jl .Lad_out_3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	movdqa 0x30(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	pxor MSG, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	jl .Lad_out_4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	movdqa 0x40(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	pxor MSG, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	jl .Lad_out_0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	jmp .Lad_a_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) .Lad_u_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	movdqu 0x00(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	jl .Lad_out_1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	movdqu 0x10(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	pxor MSG, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	jl .Lad_out_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	movdqu 0x20(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	pxor MSG, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	jl .Lad_out_3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	movdqu 0x30(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	pxor MSG, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	jl .Lad_out_4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	movdqu 0x40(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	pxor MSG, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	jl .Lad_out_0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	jmp .Lad_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) .Lad_out_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	movdqu STATE0, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	movdqu STATE1, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	movdqu STATE2, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	movdqu STATE3, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	movdqu STATE4, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) .Lad_out_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	movdqu STATE4, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	movdqu STATE0, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	movdqu STATE1, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 	movdqu STATE2, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	movdqu STATE3, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) .Lad_out_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	movdqu STATE3, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	movdqu STATE4, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	movdqu STATE0, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	movdqu STATE1, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	movdqu STATE2, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) .Lad_out_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	movdqu STATE2, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	movdqu STATE3, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	movdqu STATE4, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	movdqu STATE0, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	movdqu STATE1, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) .Lad_out_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	movdqu STATE1, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	movdqu STATE2, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	movdqu STATE3, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	movdqu STATE4, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	movdqu STATE0, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) .Lad_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) SYM_FUNC_END(crypto_aegis128_aesni_ad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) .macro encrypt_block a s0 s1 s2 s3 s4 i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	movdq\a (\i * 0x10)(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	movdqa MSG, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	pxor \s1, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	pxor \s4, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	movdqa \s2, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	pand \s3, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	pxor T1, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	movdq\a T0, (\i * 0x10)(DST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	pxor MSG, \s4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	jl .Lenc_out_\i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)  * void crypto_aegis128_aesni_enc(void *state, unsigned int length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)  *                                const void *src, void *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) SYM_FUNC_START(crypto_aegis128_aesni_enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	jb .Lenc_out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 	mov  SRC,  %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	or   DST,  %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	and $0xF, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	jnz .Lenc_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) .Lenc_a_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 	encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 	encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 	encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 	add $0x50, DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 	jmp .Lenc_a_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) .Lenc_u_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 	encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 	encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	add $0x50, DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	jmp .Lenc_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) .Lenc_out_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	movdqu STATE4, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	movdqu STATE0, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	movdqu STATE1, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 	movdqu STATE2, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	movdqu STATE3, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) .Lenc_out_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	movdqu STATE3, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 	movdqu STATE4, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	movdqu STATE0, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 	movdqu STATE1, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	movdqu STATE2, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) .Lenc_out_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	movdqu STATE2, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	movdqu STATE3, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	movdqu STATE4, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	movdqu STATE0, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 	movdqu STATE1, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) .Lenc_out_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 	movdqu STATE1, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 	movdqu STATE2, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 	movdqu STATE3, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 	movdqu STATE4, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 	movdqu STATE0, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) .Lenc_out_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 	movdqu STATE0, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 	movdqu STATE1, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 	movdqu STATE2, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 	movdqu STATE3, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 	movdqu STATE4, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) .Lenc_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) SYM_FUNC_END(crypto_aegis128_aesni_enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)  * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)  *                                     const void *src, void *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	/* encrypt message: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 	call __load_partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) 	movdqa MSG, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 	pxor STATE1, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 	pxor STATE4, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 	movdqa STATE2, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) 	pand STATE3, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) 	pxor T1, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) 	call __store_partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) 	pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 	movdqu STATE4, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) 	movdqu STATE0, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 	movdqu STATE1, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 	movdqu STATE2, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 	movdqu STATE3, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) SYM_FUNC_END(crypto_aegis128_aesni_enc_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) .macro decrypt_block a s0 s1 s2 s3 s4 i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 	movdq\a (\i * 0x10)(SRC), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) 	pxor \s1, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) 	pxor \s4, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) 	movdqa \s2, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) 	pand \s3, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) 	pxor T1, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) 	movdq\a MSG, (\i * 0x10)(DST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) 	pxor MSG, \s4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) 	sub $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) 	jl .Ldec_out_\i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)  * void crypto_aegis128_aesni_dec(void *state, unsigned int length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)  *                                const void *src, void *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) SYM_FUNC_START(crypto_aegis128_aesni_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) 	cmp $0x10, LEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 	jb .Ldec_out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) 	mov  SRC, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) 	or   DST, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) 	and $0xF, %r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) 	jnz .Ldec_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) .Ldec_a_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) 	decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) 	decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) 	decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) 	decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) 	decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) 	add $0x50, DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) 	jmp .Ldec_a_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .align 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) .Ldec_u_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) 	decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) 	decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) 	decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) 	decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) 	decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) 	add $0x50, SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) 	add $0x50, DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) 	jmp .Ldec_u_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) .Ldec_out_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) 	movdqu STATE4, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) 	movdqu STATE0, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) 	movdqu STATE1, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) 	movdqu STATE2, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) 	movdqu STATE3, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) .Ldec_out_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) 	movdqu STATE3, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) 	movdqu STATE4, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) 	movdqu STATE0, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) 	movdqu STATE1, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) 	movdqu STATE2, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) .Ldec_out_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) 	movdqu STATE2, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) 	movdqu STATE3, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) 	movdqu STATE4, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) 	movdqu STATE0, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) 	movdqu STATE1, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) .Ldec_out_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) 	movdqu STATE1, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) 	movdqu STATE2, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) 	movdqu STATE3, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) 	movdqu STATE4, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) 	movdqu STATE0, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) .Ldec_out_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) 	movdqu STATE0, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) 	movdqu STATE1, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) 	movdqu STATE2, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) 	movdqu STATE3, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) 	movdqu STATE4, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) .Ldec_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) SYM_FUNC_END(crypto_aegis128_aesni_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)  * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)  *                                     const void *src, void *dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) 	/* decrypt message: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) 	call __load_partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) 	pxor STATE1, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) 	pxor STATE4, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) 	movdqa STATE2, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) 	pand STATE3, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) 	pxor T1, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) 	movdqa MSG, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) 	call __store_partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) 	/* mask with byte count: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) 	movq LEN, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) 	punpcklbw T0, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) 	punpcklbw T0, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) 	punpcklbw T0, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) 	punpcklbw T0, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) 	movdqa .Laegis128_counter, T1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) 	pcmpgtb T1, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) 	pand T0, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) 	aegis128_update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) 	pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) 	/* store the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) 	movdqu STATE4, 0x00(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) 	movdqu STATE0, 0x10(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) 	movdqu STATE1, 0x20(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) 	movdqu STATE2, 0x30(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) 	movdqu STATE3, 0x40(STATEP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) SYM_FUNC_END(crypto_aegis128_aesni_dec_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)  * void crypto_aegis128_aesni_final(void *state, void *tag_xor,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)  *                                  u64 assoclen, u64 cryptlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) SYM_FUNC_START(crypto_aegis128_aesni_final)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) 	FRAME_BEGIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) 	/* load the state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) 	movdqu 0x00(STATEP), STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) 	movdqu 0x10(STATEP), STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) 	movdqu 0x20(STATEP), STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) 	movdqu 0x30(STATEP), STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) 	movdqu 0x40(STATEP), STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) 	/* prepare length block: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) 	movq %rdx, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) 	movq %rcx, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) 	pslldq $8, T0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) 	pxor T0, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) 	psllq $3, MSG /* multiply by 8 (to get bit count) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) 	pxor STATE3, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) 	/* update state: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) 	aegis128_update; pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) 	aegis128_update; pxor MSG, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) 	aegis128_update; pxor MSG, STATE2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) 	aegis128_update; pxor MSG, STATE1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) 	aegis128_update; pxor MSG, STATE0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) 	aegis128_update; pxor MSG, STATE4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) 	aegis128_update; pxor MSG, STATE3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) 	/* xor tag: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) 	movdqu (%rsi), MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) 	pxor STATE0, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) 	pxor STATE1, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) 	pxor STATE2, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) 	pxor STATE3, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) 	pxor STATE4, MSG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) 	movdqu MSG, (%rsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) 	FRAME_END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) SYM_FUNC_END(crypto_aegis128_aesni_final)