^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2014 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/neon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <crypto/sha.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <crypto/sha256_base.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) MODULE_ALIAS_CRYPTO("sha224");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) MODULE_ALIAS_CRYPTO("sha256");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) struct sha256_ce_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct sha256_state sst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) u32 finalize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) extern const u32 sha256_ce_offsetof_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) extern const u32 sha256_ce_offsetof_finalize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) asmlinkage int sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static void __sha2_ce_transform(struct sha256_state *sst, u8 const *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) int blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) while (blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) int rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) rem = sha2_ce_transform(container_of(sst, struct sha256_ce_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) sst), src, blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) src += (blocks - rem) * SHA256_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) blocks = rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) sst.count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) finalize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) asmlinkage void sha256_block_data_order(u32 *digest, u8 const *src, int blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static void __sha256_block_data_order(struct sha256_state *sst, u8 const *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) int blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) sha256_block_data_order(sst->state, src, blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static int sha256_ce_update(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct sha256_ce_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) if (!crypto_simd_usable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return sha256_base_do_update(desc, data, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) __sha256_block_data_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) sctx->finalize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) sha256_base_do_update(desc, data, len, __sha2_ce_transform);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned int len, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct sha256_ce_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) sha256_base_do_update(desc, data, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) __sha256_block_data_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) sha256_base_do_finalize(desc, __sha256_block_data_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return sha256_base_finish(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Allow the asm code to perform the finalization if there is no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * partial data and the input is a round multiple of the block size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) sctx->finalize = finalize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) sha256_base_do_update(desc, data, len, __sha2_ce_transform);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (!finalize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) sha256_base_do_finalize(desc, __sha2_ce_transform);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return sha256_base_finish(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static int sha256_ce_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct sha256_ce_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) sha256_base_do_finalize(desc, __sha256_block_data_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return sha256_base_finish(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) sctx->finalize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) sha256_base_do_finalize(desc, __sha2_ce_transform);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return sha256_base_finish(desc, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static int sha256_ce_export(struct shash_desc *desc, void *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct sha256_ce_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) memcpy(out, &sctx->sst, sizeof(struct sha256_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static int sha256_ce_import(struct shash_desc *desc, const void *in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct sha256_ce_state *sctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) memcpy(&sctx->sst, in, sizeof(struct sha256_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) sctx->finalize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static struct shash_alg algs[] = { {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) .init = sha224_base_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) .update = sha256_ce_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) .final = sha256_ce_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) .finup = sha256_ce_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) .export = sha256_ce_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .import = sha256_ce_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .descsize = sizeof(struct sha256_ce_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .statesize = sizeof(struct sha256_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) .digestsize = SHA224_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .cra_name = "sha224",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .cra_driver_name = "sha224-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .cra_blocksize = SHA256_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .init = sha256_base_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .update = sha256_ce_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .final = sha256_ce_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .finup = sha256_ce_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .export = sha256_ce_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .import = sha256_ce_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .descsize = sizeof(struct sha256_ce_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) .statesize = sizeof(struct sha256_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .digestsize = SHA256_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) .cra_name = "sha256",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .cra_driver_name = "sha256-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .cra_blocksize = SHA256_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) } };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static int __init sha2_ce_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return crypto_register_shashes(algs, ARRAY_SIZE(algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static void __exit sha2_ce_mod_fini(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) module_cpu_feature_match(SHA2, sha2_ce_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) module_exit(sha2_ce_mod_fini);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags