^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * it under the terms of the GNU General Public License version 2 as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/hwcap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/neon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <crypto/sha3.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) MODULE_ALIAS_CRYPTO("sha3-224");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) MODULE_ALIAS_CRYPTO("sha3-256");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) MODULE_ALIAS_CRYPTO("sha3-384");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) MODULE_ALIAS_CRYPTO("sha3-512");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) int md_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static int sha3_update(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct sha3_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) if (!crypto_simd_usable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return crypto_sha3_update(desc, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if ((sctx->partial + len) >= sctx->rsiz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) int blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (sctx->partial) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) int p = sctx->rsiz - sctx->partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) memcpy(sctx->buf + sctx->partial, data, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) data += p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) len -= p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) sctx->partial = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) blocks = len / sctx->rsiz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) len %= sctx->rsiz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) while (blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) rem = sha3_ce_transform(sctx->st, data, blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) data += (blocks - rem) * sctx->rsiz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) blocks = rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) memcpy(sctx->buf + sctx->partial, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) sctx->partial += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static int sha3_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct sha3_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) __le64 *digest = (__le64 *)out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (!crypto_simd_usable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return crypto_sha3_final(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) sctx->buf[sctx->partial++] = 0x06;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) sctx->buf[sctx->rsiz - 1] |= 0x80;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) for (i = 0; i < digest_size / 8; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) put_unaligned_le64(sctx->st[i], digest++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (digest_size & 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) put_unaligned_le32(sctx->st[i], (__le32 *)digest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *sctx = (struct sha3_state){};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static struct shash_alg algs[] = { {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) .digestsize = SHA3_224_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .init = crypto_sha3_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) .update = sha3_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) .final = sha3_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) .descsize = sizeof(struct sha3_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) .base.cra_name = "sha3-224",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) .base.cra_driver_name = "sha3-224-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) .base.cra_blocksize = SHA3_224_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) .base.cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) .base.cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) .digestsize = SHA3_256_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) .init = crypto_sha3_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) .update = sha3_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) .final = sha3_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) .descsize = sizeof(struct sha3_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) .base.cra_name = "sha3-256",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) .base.cra_driver_name = "sha3-256-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) .base.cra_blocksize = SHA3_256_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) .base.cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) .base.cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) .digestsize = SHA3_384_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) .init = crypto_sha3_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) .update = sha3_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) .final = sha3_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) .descsize = sizeof(struct sha3_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) .base.cra_name = "sha3-384",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) .base.cra_driver_name = "sha3-384-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) .base.cra_blocksize = SHA3_384_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) .base.cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) .base.cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .digestsize = SHA3_512_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .init = crypto_sha3_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) .update = sha3_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .final = sha3_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .descsize = sizeof(struct sha3_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .base.cra_name = "sha3-512",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .base.cra_driver_name = "sha3-512-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .base.cra_blocksize = SHA3_512_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) .base.cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .base.cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) } };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static int __init sha3_neon_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return crypto_register_shashes(algs, ARRAY_SIZE(algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static void __exit sha3_neon_mod_fini(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) module_cpu_feature_match(SHA3, sha3_neon_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) module_exit(sha3_neon_mod_fini);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags