^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * instructions. This file contains glue code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2009 Intel Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Huang Ying <ying.huang@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <crypto/algapi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <crypto/cryptd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <crypto/gf128mul.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define GHASH_BLOCK_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define GHASH_DIGEST_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) void clmul_ghash_mul(char *dst, const u128 *shash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) const u128 *shash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct ghash_async_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) struct cryptd_ahash *cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct ghash_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) u128 shash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct ghash_desc_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) u8 buffer[GHASH_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) u32 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static int ghash_init(struct shash_desc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) memset(dctx, 0, sizeof(*dctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static int ghash_setkey(struct crypto_shash *tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) const u8 *key, unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) be128 *x = (be128 *)key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) u64 a, b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (keylen != GHASH_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* perform multiplication by 'x' in GF(2^128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) a = be64_to_cpu(x->a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) b = be64_to_cpu(x->b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) ctx->shash.a = (b << 1) | (a >> 63);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ctx->shash.b = (a << 1) | (b >> 63);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (a >> 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) ctx->shash.b ^= ((u64)0xc2) << 56;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static int ghash_update(struct shash_desc *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) const u8 *src, unsigned int srclen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u8 *dst = dctx->buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (dctx->bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int n = min(srclen, dctx->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) dctx->bytes -= n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) srclen -= n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) while (n--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) *pos++ ^= *src++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (!dctx->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) clmul_ghash_mul(dst, &ctx->shash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) clmul_ghash_update(dst, src, srclen, &ctx->shash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (srclen & 0xf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) src += srclen - (srclen & 0xf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) srclen &= 0xf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) dctx->bytes = GHASH_BLOCK_SIZE - srclen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) while (srclen--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *dst++ ^= *src++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) u8 *dst = dctx->buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (dctx->bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) while (dctx->bytes--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) *tmp++ ^= 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) clmul_ghash_mul(dst, &ctx->shash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) dctx->bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static int ghash_final(struct shash_desc *desc, u8 *dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) u8 *buf = dctx->buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) ghash_flush(ctx, dctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) memcpy(dst, buf, GHASH_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static struct shash_alg ghash_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .digestsize = GHASH_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .init = ghash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) .update = ghash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .final = ghash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .setkey = ghash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .descsize = sizeof(struct ghash_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .cra_name = "__ghash",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) .cra_driver_name = "__ghash-pclmulqdqni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .cra_priority = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .cra_blocksize = GHASH_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .cra_ctxsize = sizeof(struct ghash_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static int ghash_async_init(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) desc->tfm = child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return crypto_shash_init(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static int ghash_async_update(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return crypto_ahash_update(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return shash_ahash_update(req, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static int ghash_async_final(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return crypto_ahash_final(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return crypto_shash_final(desc, req->result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static int ghash_async_import(struct ahash_request *req, const void *in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ghash_async_init(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) memcpy(dctx, in, sizeof(*dctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static int ghash_async_export(struct ahash_request *req, void *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) memcpy(out, dctx, sizeof(*dctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static int ghash_async_digest(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct ahash_request *cryptd_req = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (!crypto_simd_usable() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) memcpy(cryptd_req, req, sizeof(*req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) return crypto_ahash_digest(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) desc->tfm = child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return shash_ahash_digest(req, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct crypto_ahash *child = &ctx->cryptd_tfm->base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) & CRYPTO_TFM_REQ_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return crypto_ahash_setkey(child, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) static int ghash_async_init_tfm(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct cryptd_ahash *cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) CRYPTO_ALG_INTERNAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (IS_ERR(cryptd_tfm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return PTR_ERR(cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) ctx->cryptd_tfm = cryptd_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) sizeof(struct ahash_request) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) crypto_ahash_reqsize(&cryptd_tfm->base));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) cryptd_free_ahash(ctx->cryptd_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static struct ahash_alg ghash_async_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) .init = ghash_async_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) .update = ghash_async_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) .final = ghash_async_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) .setkey = ghash_async_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) .digest = ghash_async_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) .export = ghash_async_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) .import = ghash_async_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) .digestsize = GHASH_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) .statesize = sizeof(struct ghash_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) .cra_name = "ghash",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) .cra_driver_name = "ghash-clmulni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) .cra_ctxsize = sizeof(struct ghash_async_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) .cra_flags = CRYPTO_ALG_ASYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) .cra_blocksize = GHASH_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) .cra_init = ghash_async_init_tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) .cra_exit = ghash_async_exit_tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static const struct x86_cpu_id pcmul_cpu_id[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static int __init ghash_pclmulqdqni_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (!x86_match_cpu(pcmul_cpu_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) err = crypto_register_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) goto err_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) err = crypto_register_ahash(&ghash_async_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) goto err_shash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) err_shash:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) crypto_unregister_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) err_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) static void __exit ghash_pclmulqdqni_mod_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) crypto_unregister_ahash(&ghash_async_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) crypto_unregister_shash(&ghash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) module_init(ghash_pclmulqdqni_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) module_exit(ghash_pclmulqdqni_mod_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) MODULE_DESCRIPTION("GHASH hash function, accelerated by PCLMULQDQ-NI");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) MODULE_ALIAS_CRYPTO("ghash");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags