^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * amlogic-cipher.c - hardware cryptographic offloader for Amlogic GXL SoC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2018-2019 Corentin LABBE <clabbe@baylibre.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This file add support for AES cipher with 128,192,256 bits keysize in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * CBC and ECB mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <crypto/scatterwalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <crypto/internal/skcipher.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "amlogic-gxl.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static int get_engine_number(struct meson_dev *mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) return atomic_inc_return(&mc->flow) % MAXFLOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static bool meson_cipher_need_fallback(struct skcipher_request *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct scatterlist *src_sg = areq->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct scatterlist *dst_sg = areq->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) if (areq->cryptlen == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (sg_nents(src_sg) != sg_nents(dst_sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* KEY/IV descriptors use 3 desc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (sg_nents(src_sg) > MAXDESC - 3 || sg_nents(dst_sg) > MAXDESC - 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) while (src_sg && dst_sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if ((src_sg->length % 16) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if ((dst_sg->length % 16) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) if (src_sg->length != dst_sg->length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (!IS_ALIGNED(src_sg->offset, sizeof(u32)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) if (!IS_ALIGNED(dst_sg->offset, sizeof(u32)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) src_sg = sg_next(src_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) dst_sg = sg_next(dst_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static int meson_cipher_do_fallback(struct skcipher_request *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct meson_alg_template *algt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) algt = container_of(alg, struct meson_alg_template, alg.skcipher);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) algt->stat_fb++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) areq->base.complete, areq->base.data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) areq->cryptlen, areq->iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) if (rctx->op_dir == MESON_DECRYPT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) err = crypto_skcipher_decrypt(&rctx->fallback_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) err = crypto_skcipher_encrypt(&rctx->fallback_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static int meson_cipher(struct skcipher_request *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct meson_dev *mc = op->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct meson_alg_template *algt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) int flow = rctx->flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned int todo, eat, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) struct scatterlist *src_sg = areq->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct scatterlist *dst_sg = areq->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) struct meson_desc *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) int nr_sgs, nr_sgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) int i, err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned int keyivlen, ivsize, offset, tloffset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) dma_addr_t phykeyiv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) void *backup_iv = NULL, *bkeyiv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) u32 v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) algt = container_of(alg, struct meson_alg_template, alg.skcipher);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) dev_dbg(mc->dev, "%s %s %u %x IV(%u) key=%u flow=%d\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) crypto_tfm_alg_name(areq->base.tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) areq->cryptlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) rctx->op_dir, crypto_skcipher_ivsize(tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) op->keylen, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) algt->stat_req++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) mc->chanlist[flow].stat_req++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * The hardware expect a list of meson_desc structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * The 2 first structures store key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * The third stores IV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) bkeyiv = kzalloc(48, GFP_KERNEL | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (!bkeyiv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) memcpy(bkeyiv, op->key, op->keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) keyivlen = op->keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) ivsize = crypto_skcipher_ivsize(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (areq->iv && ivsize > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (ivsize > areq->cryptlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) dev_err(mc->dev, "invalid ivsize=%d vs len=%d\n", ivsize, areq->cryptlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) memcpy(bkeyiv + 32, areq->iv, ivsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) keyivlen = 48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (rctx->op_dir == MESON_DECRYPT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) backup_iv = kzalloc(ivsize, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (!backup_iv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) offset = areq->cryptlen - ivsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) scatterwalk_map_and_copy(backup_iv, areq->src, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ivsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (keyivlen == 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) keyivlen = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) phykeyiv = dma_map_single(mc->dev, bkeyiv, keyivlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) err = dma_mapping_error(mc->dev, phykeyiv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) dev_err(mc->dev, "Cannot DMA MAP KEY IV\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) tloffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) eat = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) while (keyivlen > eat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) desc = &mc->chanlist[flow].tl[tloffset];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) memset(desc, 0, sizeof(struct meson_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) todo = min(keyivlen - eat, 16u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) desc->t_src = cpu_to_le32(phykeyiv + i * 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) desc->t_dst = cpu_to_le32(i * 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) v = (MODE_KEY << 20) | DESC_OWN | 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) desc->t_status = cpu_to_le32(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) eat += todo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) tloffset++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (areq->src == areq->dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (nr_sgs < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) nr_sgd = nr_sgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (nr_sgs < 0 || nr_sgs > MAXDESC - 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (nr_sgd < 0 || nr_sgd > MAXDESC - 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) goto theend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) src_sg = areq->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) dst_sg = areq->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) len = areq->cryptlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) while (src_sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) desc = &mc->chanlist[flow].tl[tloffset];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) memset(desc, 0, sizeof(struct meson_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) desc->t_src = cpu_to_le32(sg_dma_address(src_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) desc->t_dst = cpu_to_le32(sg_dma_address(dst_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) todo = min(len, sg_dma_len(src_sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) v = (op->keymode << 20) | DESC_OWN | todo | (algt->blockmode << 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (rctx->op_dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) v |= DESC_ENCRYPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) len -= todo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (!sg_next(src_sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) v |= DESC_LAST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) desc->t_status = cpu_to_le32(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) tloffset++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) src_sg = sg_next(src_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) dst_sg = sg_next(dst_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) reinit_completion(&mc->chanlist[flow].complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) mc->chanlist[flow].status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) writel(mc->chanlist[flow].t_phy | 2, mc->base + (flow << 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) wait_for_completion_interruptible_timeout(&mc->chanlist[flow].complete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) msecs_to_jiffies(500));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (mc->chanlist[flow].status == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) dev_err(mc->dev, "DMA timeout for flow %d\n", flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (areq->src == areq->dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) dma_unmap_sg(mc->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) dma_unmap_sg(mc->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) dma_unmap_sg(mc->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (areq->iv && ivsize > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (rctx->op_dir == MESON_DECRYPT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) memcpy(areq->iv, backup_iv, ivsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) scatterwalk_map_and_copy(areq->iv, areq->dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) areq->cryptlen - ivsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) ivsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) theend:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) kfree_sensitive(bkeyiv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) kfree_sensitive(backup_iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static int meson_handle_cipher_request(struct crypto_engine *engine,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) void *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) err = meson_cipher(breq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) crypto_finalize_skcipher_request(engine, breq, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) int meson_skdecrypt(struct skcipher_request *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct crypto_engine *engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) rctx->op_dir = MESON_DECRYPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (meson_cipher_need_fallback(areq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return meson_cipher_do_fallback(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) e = get_engine_number(op->mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) engine = op->mc->chanlist[e].engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) rctx->flow = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return crypto_transfer_skcipher_request_to_engine(engine, areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) int meson_skencrypt(struct skcipher_request *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct crypto_engine *engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) int e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) rctx->op_dir = MESON_ENCRYPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (meson_cipher_need_fallback(areq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return meson_cipher_do_fallback(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) e = get_engine_number(op->mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) engine = op->mc->chanlist[e].engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) rctx->flow = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return crypto_transfer_skcipher_request_to_engine(engine, areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) int meson_cipher_init(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct meson_alg_template *algt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) const char *name = crypto_tfm_alg_name(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) memset(op, 0, sizeof(struct meson_cipher_tfm_ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) algt = container_of(alg, struct meson_alg_template, alg.skcipher);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) op->mc = algt->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (IS_ERR(op->fallback_tfm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) dev_err(op->mc->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) name, PTR_ERR(op->fallback_tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return PTR_ERR(op->fallback_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) sktfm->reqsize = sizeof(struct meson_cipher_req_ctx) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) crypto_skcipher_reqsize(op->fallback_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) op->enginectx.op.do_one_request = meson_handle_cipher_request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) op->enginectx.op.prepare_request = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) op->enginectx.op.unprepare_request = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) void meson_cipher_exit(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) kfree_sensitive(op->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) crypto_free_skcipher(op->fallback_tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct meson_dev *mc = op->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) switch (keylen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) case 128 / 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) op->keymode = MODE_AES_128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) case 192 / 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) op->keymode = MODE_AES_192;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) case 256 / 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) op->keymode = MODE_AES_256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) kfree_sensitive(op->key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) op->keylen = keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (!op->key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags