^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0 OR MIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <crypto/internal/blake2s.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/fpu/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) const u8 *block, const size_t nblocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) const u32 inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) asmlinkage void blake2s_compress_avx512(struct blake2s_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) const u8 *block, const size_t nblocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) const u32 inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) void blake2s_compress_arch(struct blake2s_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) const u8 *block, size_t nblocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) const u32 inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* SIMD disables preemption, so relax after processing each page. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) if (!static_branch_likely(&blake2s_use_ssse3) || !crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) blake2s_compress_generic(state, block, nblocks, inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) const size_t blocks = min_t(size_t, nblocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) SZ_4K / BLAKE2S_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) if (IS_ENABLED(CONFIG_AS_AVX512) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static_branch_likely(&blake2s_use_avx512))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) blake2s_compress_avx512(state, block, blocks, inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) blake2s_compress_ssse3(state, block, blocks, inc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) nblocks -= blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) block += blocks * BLAKE2S_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) } while (nblocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) EXPORT_SYMBOL(blake2s_compress_arch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static int crypto_blake2s_update_x86(struct shash_desc *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) const u8 *in, unsigned int inlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return crypto_blake2s_final(desc, out, blake2s_compress_arch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define BLAKE2S_ALG(name, driver_name, digest_size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) .base.cra_name = name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) .base.cra_driver_name = driver_name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) .base.cra_priority = 200, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .base.cra_module = THIS_MODULE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) .digestsize = digest_size, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) .setkey = crypto_blake2s_setkey, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) .init = crypto_blake2s_init, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) .update = crypto_blake2s_update_x86, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .final = crypto_blake2s_final_x86, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) .descsize = sizeof(struct blake2s_state), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static struct shash_alg blake2s_algs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static int __init blake2s_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (!boot_cpu_has(X86_FEATURE_SSSE3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static_branch_enable(&blake2s_use_ssse3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (IS_ENABLED(CONFIG_AS_AVX512) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) boot_cpu_has(X86_FEATURE_AVX) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) boot_cpu_has(X86_FEATURE_AVX2) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) boot_cpu_has(X86_FEATURE_AVX512F) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) boot_cpu_has(X86_FEATURE_AVX512VL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) XFEATURE_MASK_AVX512, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static_branch_enable(&blake2s_use_avx512);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return IS_REACHABLE(CONFIG_CRYPTO_HASH) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) crypto_register_shashes(blake2s_algs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ARRAY_SIZE(blake2s_algs)) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static void __exit blake2s_mod_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (IS_REACHABLE(CONFIG_CRYPTO_HASH) && boot_cpu_has(X86_FEATURE_SSSE3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) crypto_unregister_shashes(blake2s_algs, ARRAY_SIZE(blake2s_algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) module_init(blake2s_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) module_exit(blake2s_mod_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) MODULE_ALIAS_CRYPTO("blake2s-128");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) MODULE_ALIAS_CRYPTO("blake2s-128-x86");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) MODULE_ALIAS_CRYPTO("blake2s-160");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) MODULE_ALIAS_CRYPTO("blake2s-160-x86");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) MODULE_ALIAS_CRYPTO("blake2s-224");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) MODULE_ALIAS_CRYPTO("blake2s-224-x86");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) MODULE_ALIAS_CRYPTO("blake2s-256");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) MODULE_ALIAS_CRYPTO("blake2s-256-x86");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) MODULE_LICENSE("GPL v2");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags