Orange Pi5 kernel

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * BLAKE2b digest algorithm, NEON accelerated
 *
 * Copyright 2020 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */
 
#include <linux/linkage.h>
 
<------>.text
<------>.fpu		neon
 
<------>// The arguments to blake2b_compress_neon()
<------>STATE		.req	r0
<------>BLOCK		.req	r1
<------>NBLOCKS		.req	r2
<------>INC		.req	r3
 
<------>// Pointers to the rotation tables
<------>ROR24_TABLE	.req	r4
<------>ROR16_TABLE	.req	r5
 
<------>// The original stack pointer
<------>ORIG_SP		.req	r6
 
<------>// NEON registers which contain the message words of the current block.
<------>// M_0-M_3 are occasionally used for other purposes too.
<------>M_0		.req	d16
<------>M_1		.req	d17
<------>M_2		.req	d18
<------>M_3		.req	d19
<------>M_4		.req	d20
<------>M_5		.req	d21
<------>M_6		.req	d22
<------>M_7		.req	d23
<------>M_8		.req	d24
<------>M_9		.req	d25
<------>M_10		.req	d26
<------>M_11		.req	d27
<------>M_12		.req	d28
<------>M_13		.req	d29
<------>M_14		.req	d30
<------>M_15		.req	d31
 
<------>.align		4
<------>// Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8
<------>// instruction.  This is the most efficient way to implement these
<------>// rotation amounts with NEON.  (On Cortex-A53 it's the same speed as
<------>// vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.)
.Lror24_table:
<------>.byte		3, 4, 5, 6, 7, 0, 1, 2
.Lror16_table:
<------>.byte		2, 3, 4, 5, 6, 7, 0, 1
<------>// The BLAKE2b initialization vector
.Lblake2b_IV:
<------>.quad		0x6a09e667f3bcc908, 0xbb67ae8584caa73b
<------>.quad		0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1
<------>.quad		0x510e527fade682d1, 0x9b05688c2b3e6c1f
<------>.quad		0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
 
// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the
// NEON registers q0-q7.  The message block is in q8..q15 (M_0-M_15).  The stack
// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9
// (M_0-M_3), so that they can be reloaded if they are used as temporary
// registers.  The macro arguments s0-s15 give the order in which the message
// words are used in this round.  'final' is 1 if this is the final round.
.macro	_blake2b_round	s0, s1, s2, s3, s4, s5, s6, s7, \
<------><------><------>s8, s9, s10, s11, s12, s13, s14, s15, final=0
 
<------>// Mix the columns:
<------>// (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]),
<------>// (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]).
 
<------>// a += b + m[blake2b_sigma[r][2*i + 0]];
<------>vadd.u64	q0, q0, q2
<------>vadd.u64	q1, q1, q3
<------>vadd.u64	d0, d0, M_\s0
<------>vadd.u64	d1, d1, M_\s2
<------>vadd.u64	d2, d2, M_\s4
<------>vadd.u64	d3, d3, M_\s6
 
<------>// d = ror64(d ^ a, 32);
<------>veor		q6, q6, q0
<------>veor		q7, q7, q1
<------>vrev64.32	q6, q6
<------>vrev64.32	q7, q7
 
<------>// c += d;
<------>vadd.u64	q4, q4, q6
<------>vadd.u64	q5, q5, q7
 
<------>// b = ror64(b ^ c, 24);
<------>vld1.8		{M_0}, [ROR24_TABLE, :64]
<------>veor		q2, q2, q4
<------>veor		q3, q3, q5
<------>vtbl.8		d4, {d4}, M_0
<------>vtbl.8		d5, {d5}, M_0
<------>vtbl.8		d6, {d6}, M_0
<------>vtbl.8		d7, {d7}, M_0
 
<------>// a += b + m[blake2b_sigma[r][2*i + 1]];
<------>//
<------>// M_0 got clobbered above, so we have to reload it if any of the four
<------>// message words this step needs happens to be M_0.  Otherwise we don't
<------>// need to reload it here, as it will just get clobbered again below.
.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0
<------>vld1.8		{M_0}, [sp, :64]
.endif
<------>vadd.u64	q0, q0, q2
<------>vadd.u64	q1, q1, q3
<------>vadd.u64	d0, d0, M_\s1
<------>vadd.u64	d1, d1, M_\s3
<------>vadd.u64	d2, d2, M_\s5
<------>vadd.u64	d3, d3, M_\s7
 
<------>// d = ror64(d ^ a, 16);
<------>vld1.8		{M_0}, [ROR16_TABLE, :64]
<------>veor		q6, q6, q0
<------>veor		q7, q7, q1
<------>vtbl.8		d12, {d12}, M_0
<------>vtbl.8		d13, {d13}, M_0
<------>vtbl.8		d14, {d14}, M_0
<------>vtbl.8		d15, {d15}, M_0
 
<------>// c += d;
<------>vadd.u64	q4, q4, q6
<------>vadd.u64	q5, q5, q7
 
<------>// b = ror64(b ^ c, 63);
<------>//
<------>// This rotation amount isn't a multiple of 8, so it has to be
<------>// implemented using a pair of shifts, which requires temporary
<------>// registers.  Use q8-q9 (M_0-M_3) for this, and reload them afterwards.
<------>veor		q8, q2, q4
<------>veor		q9, q3, q5
<------>vshr.u64	q2, q8, #63
<------>vshr.u64	q3, q9, #63
<------>vsli.u64	q2, q8, #1
<------>vsli.u64	q3, q9, #1
<------>vld1.8		{q8-q9}, [sp, :256]
 
<------>// Mix the diagonals:
<------>// (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]),
<------>// (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]).
<------>//
<------>// There are two possible ways to do this: use 'vext' instructions to
<------>// shift the rows of the matrix so that the diagonals become columns,
<------>// and undo it afterwards; or just use 64-bit operations on 'd'
<------>// registers instead of 128-bit operations on 'q' registers.  We use the
<------>// latter approach, as it performs much better on Cortex-A7.
 
<------>// a += b + m[blake2b_sigma[r][2*i + 0]];
<------>vadd.u64	d0, d0, d5
<------>vadd.u64	d1, d1, d6
<------>vadd.u64	d2, d2, d7
<------>vadd.u64	d3, d3, d4
<------>vadd.u64	d0, d0, M_\s8
<------>vadd.u64	d1, d1, M_\s10
<------>vadd.u64	d2, d2, M_\s12
<------>vadd.u64	d3, d3, M_\s14
 
<------>// d = ror64(d ^ a, 32);
<------>veor		d15, d15, d0
<------>veor		d12, d12, d1
<------>veor		d13, d13, d2
<------>veor		d14, d14, d3
<------>vrev64.32	d15, d15
<------>vrev64.32	d12, d12
<------>vrev64.32	d13, d13
<------>vrev64.32	d14, d14
 
<------>// c += d;
<------>vadd.u64	d10, d10, d15
<------>vadd.u64	d11, d11, d12
<------>vadd.u64	d8, d8, d13
<------>vadd.u64	d9, d9, d14
 
<------>// b = ror64(b ^ c, 24);
<------>vld1.8		{M_0}, [ROR24_TABLE, :64]
<------>veor		d5, d5, d10
<------>veor		d6, d6, d11
<------>veor		d7, d7, d8
<------>veor		d4, d4, d9
<------>vtbl.8		d5, {d5}, M_0
<------>vtbl.8		d6, {d6}, M_0
<------>vtbl.8		d7, {d7}, M_0
<------>vtbl.8		d4, {d4}, M_0
 
<------>// a += b + m[blake2b_sigma[r][2*i + 1]];
.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0
<------>vld1.8		{M_0}, [sp, :64]
.endif
<------>vadd.u64	d0, d0, d5
<------>vadd.u64	d1, d1, d6
<------>vadd.u64	d2, d2, d7
<------>vadd.u64	d3, d3, d4
<------>vadd.u64	d0, d0, M_\s9
<------>vadd.u64	d1, d1, M_\s11
<------>vadd.u64	d2, d2, M_\s13
<------>vadd.u64	d3, d3, M_\s15
 
<------>// d = ror64(d ^ a, 16);
<------>vld1.8		{M_0}, [ROR16_TABLE, :64]
<------>veor		d15, d15, d0
<------>veor		d12, d12, d1
<------>veor		d13, d13, d2
<------>veor		d14, d14, d3
<------>vtbl.8		d12, {d12}, M_0
<------>vtbl.8		d13, {d13}, M_0
<------>vtbl.8		d14, {d14}, M_0
<------>vtbl.8		d15, {d15}, M_0
 
<------>// c += d;
<------>vadd.u64	d10, d10, d15
<------>vadd.u64	d11, d11, d12
<------>vadd.u64	d8, d8, d13
<------>vadd.u64	d9, d9, d14
 
<------>// b = ror64(b ^ c, 63);
<------>veor		d16, d4, d9
<------>veor		d17, d5, d10
<------>veor		d18, d6, d11
<------>veor		d19, d7, d8
<------>vshr.u64	q2, q8, #63
<------>vshr.u64	q3, q9, #63
<------>vsli.u64	q2, q8, #1
<------>vsli.u64	q3, q9, #1
<------>// Reloading q8-q9 can be skipped on the final round.
.if ! \final
<------>vld1.8		{q8-q9}, [sp, :256]
.endif
.endm
 
//
// void blake2b_compress_neon(struct blake2b_state *state,
//			      const u8 *block, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2b_state are used:
//	u64 h[8];	(inout)
//	u64 t[2];	(inout)
//	u64 f[2];	(in)
//
<------>.align		5
ENTRY(blake2b_compress_neon)
<------>push		{r4-r10}
 
<------>// Allocate a 32-byte stack buffer that is 32-byte aligned.
<------>mov		ORIG_SP, sp
<------>sub		ip, sp, #32
<------>bic		ip, ip, #31
<------>mov		sp, ip
 
<------>adr		ROR24_TABLE, .Lror24_table
<------>adr		ROR16_TABLE, .Lror16_table
 
<------>mov		ip, STATE
<------>vld1.64		{q0-q1}, [ip]!		// Load h[0..3]
<------>vld1.64		{q2-q3}, [ip]!		// Load h[4..7]
.Lnext_block:
<------>  adr		r10, .Lblake2b_IV
<------>vld1.64		{q14-q15}, [ip]		// Load t[0..1] and f[0..1]
<------>vld1.64		{q4-q5}, [r10]!		// Load IV[0..3]
<------>  vmov		r7, r8, d28		// Copy t[0] to (r7, r8)
<------>vld1.64		{q6-q7}, [r10]		// Load IV[4..7]
<------>  adds		r7, r7, INC		// Increment counter
<------>bcs		.Lslow_inc_ctr
<------>vmov.i32	d28[0], r7
<------>vst1.64		{d28}, [ip]		// Update t[0]
.Linc_ctr_done:
 
<------>// Load the next message block and finish initializing the state matrix
<------>// 'v'.  Fortunately, there are exactly enough NEON registers to fit the
<------>// entire state matrix in q0-q7 and the entire message block in q8-15.
<------>//
<------>// However, _blake2b_round also needs some extra registers for rotates,
<------>// so we have to spill some registers.  It's better to spill the message
<------>// registers than the state registers, as the message doesn't change.
<------>// Therefore we store a copy of the first 32 bytes of the message block
<------>// (q8-q9) in an aligned buffer on the stack so that they can be
<------>// reloaded when needed.  (We could just reload directly from the
<------>// message buffer, but it's faster to use aligned loads.)
<------>vld1.8		{q8-q9}, [BLOCK]!
<------>  veor		q6, q6, q14	// v[12..13] = IV[4..5] ^ t[0..1]
<------>vld1.8		{q10-q11}, [BLOCK]!
<------>  veor		q7, q7, q15	// v[14..15] = IV[6..7] ^ f[0..1]
<------>vld1.8		{q12-q13}, [BLOCK]!
<------>vst1.8		{q8-q9}, [sp, :256]
<------>  mov		ip, STATE
<------>vld1.8		{q14-q15}, [BLOCK]!
 
<------>// Execute the rounds.  Each round is provided the order in which it
<------>// needs to use the message words.
<------>_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
<------>_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
<------>_blake2b_round	11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
<------>_blake2b_round	7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
<------>_blake2b_round	9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
<------>_blake2b_round	2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
<------>_blake2b_round	12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
<------>_blake2b_round	13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
<------>_blake2b_round	6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
<------>_blake2b_round	10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0
<------>_blake2b_round	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
<------>_blake2b_round	14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \
<------><------><------>final=1
 
<------>// Fold the final state matrix into the hash chaining value:
<------>//
<------>//	for (i = 0; i < 8; i++)
<------>//		h[i] ^= v[i] ^ v[i + 8];
<------>//
<------>  vld1.64	{q8-q9}, [ip]!		// Load old h[0..3]
<------>veor		q0, q0, q4		// v[0..1] ^= v[8..9]
<------>veor		q1, q1, q5		// v[2..3] ^= v[10..11]
<------>  vld1.64	{q10-q11}, [ip]		// Load old h[4..7]
<------>veor		q2, q2, q6		// v[4..5] ^= v[12..13]
<------>veor		q3, q3, q7		// v[6..7] ^= v[14..15]
<------>veor		q0, q0, q8		// v[0..1] ^= h[0..1]
<------>veor		q1, q1, q9		// v[2..3] ^= h[2..3]
<------>  mov		ip, STATE
<------>  subs		NBLOCKS, NBLOCKS, #1	// nblocks--
<------>  vst1.64	{q0-q1}, [ip]!		// Store new h[0..3]
<------>veor		q2, q2, q10		// v[4..5] ^= h[4..5]
<------>veor		q3, q3, q11		// v[6..7] ^= h[6..7]
<------>  vst1.64	{q2-q3}, [ip]!		// Store new h[4..7]
 
<------>// Advance to the next block, if there is one.
<------>bne		.Lnext_block		// nblocks != 0?
 
<------>mov		sp, ORIG_SP
<------>pop		{r4-r10}
<------>mov		pc, lr
 
.Lslow_inc_ctr:
<------>// Handle the case where the counter overflowed its low 32 bits, by
<------>// carrying the overflow bit into the full 128-bit counter.
<------>vmov		r9, r10, d29
<------>adcs		r8, r8, #0
<------>adcs		r9, r9, #0
<------>adc		r10, r10, #0
<------>vmov		d28, r7, r8
<------>vmov		d29, r9, r10
<------>vst1.64		{q14}, [ip]		// Update t[0] and t[1]
<------>b		.Linc_ctr_done
ENDPROC(blake2b_compress_neon)