Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2015 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/hwcap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/neon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <crypto/b128ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <crypto/cryptd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <crypto/gf128mul.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) MODULE_ALIAS_CRYPTO("ghash");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #define GHASH_BLOCK_SIZE	16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #define GHASH_DIGEST_SIZE	16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) struct ghash_key {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	be128	k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	u64	h[][2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) struct ghash_desc_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	u8 buf[GHASH_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	u32 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) struct ghash_async_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	struct cryptd_ahash *cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 				       u64 const h[][2], const char *head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 				      u64 const h[][2], const char *head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) static int ghash_init(struct shash_desc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	*ctx = (struct ghash_desc_ctx){};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) static void ghash_do_update(int blocks, u64 dg[], const char *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 			    struct ghash_key *key, const char *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	if (likely(crypto_simd_usable())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 		kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		if (static_branch_likely(&use_p64))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 			pmull_ghash_update_p64(blocks, dg, src, key->h, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 			pmull_ghash_update_p8(blocks, dg, src, key->h, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 			const u8 *in = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 			if (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 				in = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 				blocks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 				head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 			} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 				src += GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 			crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 			gf128mul_lle(&dst, &key->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 		} while (--blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		dg[0] = be64_to_cpu(dst.b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		dg[1] = be64_to_cpu(dst.a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) static int ghash_update(struct shash_desc *desc, const u8 *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 			unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	ctx->count += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	if ((partial + len) >= GHASH_BLOCK_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		int blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		if (partial) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 			int p = GHASH_BLOCK_SIZE - partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 			memcpy(ctx->buf + partial, src, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 			src += p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 			len -= p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		blocks = len / GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 		len %= GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 		ghash_do_update(blocks, ctx->digest, src, key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 				partial ? ctx->buf : NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		src += blocks * GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		partial = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	if (len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 		memcpy(ctx->buf + partial, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static int ghash_final(struct shash_desc *desc, u8 *dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	if (partial) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	put_unaligned_be64(ctx->digest[1], dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	put_unaligned_be64(ctx->digest[0], dst + 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	*ctx = (struct ghash_desc_ctx){};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static void ghash_reflect(u64 h[], const be128 *k)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	u64 carry = be64_to_cpu(k->a) >> 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	h[0] = (be64_to_cpu(k->b) << 1) | carry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	if (carry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		h[1] ^= 0xc200000000000000UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static int ghash_setkey(struct crypto_shash *tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 			const u8 *inkey, unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	struct ghash_key *key = crypto_shash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	if (keylen != GHASH_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	/* needed for the fallback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	ghash_reflect(key->h[0], &key->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	if (static_branch_likely(&use_p64)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 		be128 h = key->k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		gf128mul_lle(&h, &key->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		ghash_reflect(key->h[1], &h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		gf128mul_lle(&h, &key->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		ghash_reflect(key->h[2], &h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		gf128mul_lle(&h, &key->k);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		ghash_reflect(key->h[3], &h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static struct shash_alg ghash_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	.digestsize		= GHASH_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	.init			= ghash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	.update			= ghash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	.final			= ghash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	.setkey			= ghash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	.descsize		= sizeof(struct ghash_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	.base.cra_name		= "ghash",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	.base.cra_driver_name	= "ghash-ce-sync",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	.base.cra_priority	= 300 - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	.base.cra_blocksize	= GHASH_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	.base.cra_ctxsize	= sizeof(struct ghash_key) + sizeof(u64[2]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	.base.cra_module	= THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static int ghash_async_init(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	desc->tfm = child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	return crypto_shash_init(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static int ghash_async_update(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		return crypto_ahash_update(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		return shash_ahash_update(req, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static int ghash_async_final(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		return crypto_ahash_final(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		return crypto_shash_final(desc, req->result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static int ghash_async_digest(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	    (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 		memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		return crypto_ahash_digest(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		desc->tfm = child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		return shash_ahash_digest(req, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) static int ghash_async_import(struct ahash_request *req, const void *in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	return crypto_shash_import(desc, in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static int ghash_async_export(struct ahash_request *req, void *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	return crypto_shash_export(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 			      unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 			       & CRYPTO_TFM_REQ_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	return crypto_ahash_setkey(child, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static int ghash_async_init_tfm(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	struct cryptd_ahash *cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	if (IS_ERR(cryptd_tfm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 		return PTR_ERR(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	ctx->cryptd_tfm = cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 				 sizeof(struct ahash_request) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 				 crypto_ahash_reqsize(&cryptd_tfm->base));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	cryptd_free_ahash(ctx->cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static struct ahash_alg ghash_async_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	.init			= ghash_async_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	.update			= ghash_async_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	.final			= ghash_async_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	.setkey			= ghash_async_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	.digest			= ghash_async_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	.import			= ghash_async_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	.export			= ghash_async_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	.halg.digestsize	= GHASH_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	.halg.statesize		= sizeof(struct ghash_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	.halg.base		= {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		.cra_name	= "ghash",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		.cra_driver_name = "ghash-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 		.cra_priority	= 300,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 		.cra_flags	= CRYPTO_ALG_ASYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		.cra_blocksize	= GHASH_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 		.cra_ctxsize	= sizeof(struct ghash_async_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 		.cra_module	= THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 		.cra_init	= ghash_async_init_tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 		.cra_exit	= ghash_async_exit_tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static int __init ghash_ce_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	if (!(elf_hwcap & HWCAP_NEON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 		return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	if (elf_hwcap2 & HWCAP2_PMULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 		ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		static_branch_enable(&use_p64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	err = crypto_register_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	err = crypto_register_ahash(&ghash_async_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 		goto err_shash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) err_shash:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	crypto_unregister_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static void __exit ghash_ce_mod_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	crypto_unregister_ahash(&ghash_async_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	crypto_unregister_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) module_init(ghash_ce_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) module_exit(ghash_ce_mod_exit);