^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* GPL HEADER START
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * it under the terms of the GNU General Public License version 2 only,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * General Public License version 2 for more details (a copy is included
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * in the LICENSE file that accompanied this code).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * You should have received a copy of the GNU General Public License
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * version 2 along with this program; If not, see http://www.gnu.org/licenses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Please visit http://www.xyratex.com/contact if you need additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * information or have any questions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * GPL HEADER END
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Copyright 2012 Xyratex Technology Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * Wrappers for kernel crypto shash api to pclmulqdq crc32 imlementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/crc32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/cpufeatures.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define CHKSUM_BLOCK_SIZE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define CHKSUM_DIGEST_SIZE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define PCLMUL_MIN_LEN 64L /* minimum size of buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * for crc32_pclmul_le_16 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define SCALE_F 16L /* size of xmm register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define SCALE_F_MASK (SCALE_F - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static u32 __attribute__((pure))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned int iquotient;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) unsigned int iremainder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) unsigned int prealign;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) return crc32_le(crc, p, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if ((long)p & SCALE_F_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) /* align p to 16 byte */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) prealign = SCALE_F - ((long)p & SCALE_F_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) crc = crc32_le(crc, p, prealign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) len -= prealign;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ~SCALE_F_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) iquotient = len & (~SCALE_F_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) iremainder = len & SCALE_F_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) crc = crc32_pclmul_le_16(p, iquotient, crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) if (iremainder)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) crc = crc32_le(crc, p + iquotient, iremainder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) u32 *key = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) *key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) u32 *mctx = crypto_shash_ctx(hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (keylen != sizeof(u32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *mctx = le32_to_cpup((__le32 *)key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static int crc32_pclmul_init(struct shash_desc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) u32 *mctx = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) u32 *crcp = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *crcp = *mctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) u32 *crcp = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) *crcp = crc32_pclmul_le(*crcp, data, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /* No final XOR 0xFFFFFFFF, like crc32_le */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unsigned int len, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) u32 *crcp = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) *(__le32 *)out = cpu_to_le32p(crcp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned int len, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static struct shash_alg alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .setkey = crc32_pclmul_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .init = crc32_pclmul_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .update = crc32_pclmul_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .final = crc32_pclmul_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .finup = crc32_pclmul_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .digest = crc32_pclmul_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .descsize = sizeof(u32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .digestsize = CHKSUM_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) .cra_name = "crc32",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .cra_driver_name = "crc32-pclmul",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .cra_priority = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .cra_blocksize = CHKSUM_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .cra_ctxsize = sizeof(u32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) .cra_init = crc32_pclmul_cra_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static int __init crc32_pclmul_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (!x86_match_cpu(crc32pclmul_cpu_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) pr_info("PCLMULQDQ-NI instructions are not detected.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return crypto_register_shash(&alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static void __exit crc32_pclmul_mod_fini(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) crypto_unregister_shash(&alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) module_init(crc32_pclmul_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) module_exit(crc32_pclmul_mod_fini);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) MODULE_ALIAS_CRYPTO("crc32");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) MODULE_ALIAS_CRYPTO("crc32-pclmul");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags