^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * AMD Cryptographic Coprocessor (CCP) driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Tom Lendacky <thomas.lendacky@amd.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Gary R Hook <gary.hook@amd.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <crypto/scatterwalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <crypto/des.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/ccp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "ccp-dev.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /* SHA initial context values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) cpu_to_be32(SHA1_H4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ccp_gen_jobid(ccp) : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static u32 ccp_gen_jobid(struct ccp_device *ccp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static void ccp_sg_free(struct ccp_sg_workarea *wa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (wa->dma_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) wa->dma_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct scatterlist *sg, u64 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) enum dma_data_direction dma_dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) memset(wa, 0, sizeof(*wa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) wa->sg = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) wa->nents = sg_nents_for_len(sg, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (wa->nents < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return wa->nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) wa->bytes_left = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) wa->sg_used = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (len == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (dma_dir == DMA_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) wa->dma_sg = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) wa->dma_sg_head = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) wa->dma_dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) wa->dma_dir = dma_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (!wa->dma_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned int nbytes = min_t(u64, len, wa->bytes_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned int sg_combined_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (!wa->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) wa->sg_used += nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) wa->bytes_left -= nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* Advance to the next DMA scatterlist entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) wa->dma_sg = sg_next(wa->dma_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* In the case that the DMA mapped scatterlist has entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * that have been merged, the non-DMA mapped scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * must be advanced multiple times for each merged entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * This ensures that the current non-DMA mapped entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * corresponds to the current DMA mapped entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) sg_combined_len += wa->sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) wa->sg = sg_next(wa->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) } while (wa->sg_used > sg_combined_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) wa->sg_used = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void ccp_dm_free(struct ccp_dm_workarea *wa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (wa->address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) dma_pool_free(wa->dma_pool, wa->address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) wa->dma.address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (wa->dma.address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) dma_unmap_single(wa->dev, wa->dma.address, wa->length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) wa->dma.dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) kfree(wa->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) wa->address = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) wa->dma.address = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) memset(wa, 0, sizeof(*wa));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) wa->dev = cmd_q->ccp->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) wa->length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (len <= CCP_DMAPOOL_MAX_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) wa->dma_pool = cmd_q->dma_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) wa->address = dma_pool_zalloc(wa->dma_pool, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) &wa->dma.address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (!wa->address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) wa->address = kzalloc(len, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!wa->address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) wa->dma.address = dma_map_single(wa->dev, wa->address, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (dma_mapping_error(wa->dev, wa->dma.address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) wa->dma.length = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) wa->dma.dir = dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct scatterlist *sg, unsigned int sg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) WARN_ON(!wa->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (len > (wa->length - wa_offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct scatterlist *sg, unsigned int sg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) WARN_ON(!wa->address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) unsigned int wa_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) unsigned int sg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) u8 *p, *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) p = wa->address + wa_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) q = p + len - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) while (p < q) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) *p = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) *q = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) *p = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) p++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) q--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) unsigned int wa_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) unsigned int sg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) u8 *p, *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) p = wa->address + wa_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) q = p + len - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) while (p < q) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) *p = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) *q = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) *p = *p ^ *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) p++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) q--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) ccp_dm_free(&data->dm_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) ccp_sg_free(&data->sg_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct scatterlist *sg, u64 sg_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) unsigned int dm_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) memset(data, 0, sizeof(*data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) goto e_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) goto e_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) e_err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) ccp_free_data(data, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct ccp_sg_workarea *sg_wa = &data->sg_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct ccp_dm_workarea *dm_wa = &data->dm_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) unsigned int buf_count, nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Clear the buffer if setting it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (!from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) memset(dm_wa->address, 0, dm_wa->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (!sg_wa->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /* Perform the copy operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * nbytes will always be <= UINT_MAX because dm_wa->length is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * an unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) nbytes, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /* Update the structures and generate the count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) buf_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) dm_wa->length - buf_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) buf_count += nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ccp_update_sg_workarea(sg_wa, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) return buf_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return ccp_queue_buf(data, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return ccp_queue_buf(data, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct ccp_op *op, unsigned int block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) bool blocksize_op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) unsigned int sg_src_len, sg_dst_len, op_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /* The CCP can only DMA from/to one address each per operation. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * requires that we find the smallest DMA area between the source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * and destination. The resulting len values will always be <= UINT_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * because the dma length is an unsigned int.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) op_len = min(sg_src_len, sg_dst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) op_len = sg_src_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* The data operation length will be at least block_size in length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * or the smaller of available sg room remaining for the source or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * the destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) op_len = max(op_len, block_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* Unless we have to buffer data, there's no reason to wait */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) op->soc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (sg_src_len < block_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /* Not enough data in the sg element, so it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * needs to be buffered into a blocksize chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) int cp_len = ccp_fill_queue_buf(src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) op->soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) op->src.u.dma.address = src->dm_wa.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) op->src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) /* Enough data in the sg element, but we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * adjust for any previously copied data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) op->src.u.dma.offset = src->sg_wa.sg_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) op->src.u.dma.length = op_len & ~(block_size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (sg_dst_len < block_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* Not enough room in the sg element or we're on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * last piece of data (when using padding), so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * output needs to be buffered into a blocksize chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) op->soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) op->dst.u.dma.address = dst->dm_wa.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) op->dst.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) op->dst.u.dma.length = op->src.u.dma.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /* Enough room in the sg element, but we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * adjust for any previously used area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) op->dst.u.dma.offset = dst->sg_wa.sg_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) op->dst.u.dma.length = op->src.u.dma.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) op->init = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (op->dst.u.dma.address == dst->dm_wa.dma.address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) ccp_empty_queue_buf(dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) ccp_update_sg_workarea(&dst->sg_wa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) op->dst.u.dma.length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) u32 byte_swap, bool from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) op.jobid = jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (from) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) op.src.type = CCP_MEMTYPE_SB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) op.src.u.sb = sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) op.dst.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) op.dst.u.dma.address = wa->dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) op.dst.u.dma.length = wa->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) op.src.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) op.src.u.dma.address = wa->dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) op.src.u.dma.length = wa->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) op.dst.type = CCP_MEMTYPE_SB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) op.dst.u.sb = sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) op.u.passthru.byte_swap = byte_swap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return cmd_q->ccp->vdata->perform->passthru(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) u32 byte_swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) u32 byte_swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct ccp_aes_engine *aes = &cmd->u.aes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) struct ccp_dm_workarea key, ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) struct ccp_data src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) unsigned int dm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (!((aes->key_len == AES_KEYSIZE_128) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) (aes->key_len == AES_KEYSIZE_192) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) (aes->key_len == AES_KEYSIZE_256)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (aes->src_len & (AES_BLOCK_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (aes->iv_len != AES_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (!aes->key || !aes->iv || !aes->src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (aes->cmac_final) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (aes->cmac_key_len != AES_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (!aes->cmac_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) op.sb_ctx = cmd_q->sb_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) op.u.aes.type = aes->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) op.u.aes.mode = aes->mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) op.u.aes.action = aes->action;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /* All supported key sizes fit in a single (32-byte) SB entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * and must be in little endian format. Use the 256-bit byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * swap passthru option to convert from big endian to little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) ret = ccp_init_dm_workarea(&key, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) dm_offset = CCP_SB_BYTES - aes->key_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /* The AES context fits in a single (32-byte) SB entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * must be in little endian format. Use the 256-bit byte swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * passthru option to convert from big endian to little endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) ret = ccp_init_dm_workarea(&ctx, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* Send data to the CCP AES engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) AES_BLOCK_SIZE, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (aes->cmac_final && !src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* Push the K1/K2 key to the CCP now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) aes->cmac_key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) ret = cmd_q->ccp->vdata->perform->aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) ccp_process_data(&src, NULL, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) /* Retrieve the AES context - convert from LE to BE using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * 32-byte (256-bit) byteswapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) /* ...but we only need AES_BLOCK_SIZE bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) e_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) ccp_dm_free(&key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct ccp_aes_engine *aes = &cmd->u.aes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct ccp_dm_workarea key, ctx, final_wa, tag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct ccp_data src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct ccp_data aad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) unsigned int dm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) unsigned int authsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) unsigned int jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) unsigned int ilen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) bool in_place = true; /* Default value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) __be64 *final;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct scatterlist *p_inp, sg_inp[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct scatterlist *p_tag, sg_tag[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct scatterlist *p_outp, sg_outp[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct scatterlist *p_aad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (!aes->iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (!((aes->key_len == AES_KEYSIZE_128) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) (aes->key_len == AES_KEYSIZE_192) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) (aes->key_len == AES_KEYSIZE_256)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (!aes->key) /* Gotta have a key SGL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) /* Zero defaults to 16 bytes, the maximum size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) switch (authsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) case 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) case 15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) case 14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) case 13:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) case 12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* First, decompose the source buffer into AAD & PT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * and the destination buffer into AAD, CT & tag, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * the input into CT & tag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * It is expected that the input and output SGs will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * be valid, even if the AAD and input lengths are 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) p_aad = aes->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (aes->action == CCP_AES_ACTION_ENCRYPT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) ilen = aes->src_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* Input length for decryption includes tag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) ilen = aes->src_len - authsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) op.jobid = jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) op.sb_key = cmd_q->sb_key; /* Pre-allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) op.u.aes.type = aes->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /* Copy the key to the LSB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) ret = ccp_init_dm_workarea(&key, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) dm_offset = CCP_SB_BYTES - aes->key_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /* Copy the context (IV) to the LSB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * There is an assumption here that the IV is 96 bits in length, plus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) ret = ccp_init_dm_workarea(&ctx, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) if (aes->aad_len > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /* Step 1: Run a GHASH over the Additional Authenticated Data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) op.u.aes.mode = CCP_AES_MODE_GHASH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) op.u.aes.action = CCP_AES_GHASHAAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) while (aad.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) ret = cmd_q->ccp->vdata->perform->aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) goto e_aad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) ccp_process_data(&aad, NULL, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) op.init = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) op.u.aes.mode = CCP_AES_MODE_GCTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) op.u.aes.action = aes->action;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (ilen > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) /* Step 2: Run a GCTR over the plaintext */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) in_place ? DMA_BIDIRECTIONAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) : DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) goto e_aad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (in_place) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) dst = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) AES_BLOCK_SIZE, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) op.soc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) op.eom = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (!src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) unsigned int nbytes = ilen % AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (nbytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) op.u.aes.size = (nbytes * 8) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) ret = cmd_q->ccp->vdata->perform->aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) ccp_process_data(&src, &dst, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) op.init = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) /* Step 3: Update the IV portion of the context with the original IV */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) /* Step 4: Concatenate the lengths of the AAD and source, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * hash that 16 byte buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) final = (__be64 *)final_wa.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) final[0] = cpu_to_be64(aes->aad_len * 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) final[1] = cpu_to_be64(ilen * 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) op.jobid = jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) op.sb_key = cmd_q->sb_key; /* Pre-allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) op.u.aes.type = aes->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) op.u.aes.mode = CCP_AES_MODE_GHASH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) op.u.aes.action = CCP_AES_GHASHFINAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) op.src.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) op.src.u.dma.address = final_wa.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) op.src.u.dma.length = AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) op.dst.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) op.dst.u.dma.address = final_wa.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) op.dst.u.dma.length = AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) op.u.aes.size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) ret = cmd_q->ccp->vdata->perform->aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) goto e_final_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (aes->action == CCP_AES_ACTION_ENCRYPT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) /* Put the ciphered tag after the ciphertext. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) /* Does this ciphered tag match the input? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) goto e_final_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) ccp_dm_free(&tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) goto e_final_wa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) ret = crypto_memneq(tag.address, final_wa.address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) authsize) ? -EBADMSG : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) ccp_dm_free(&tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) e_final_wa:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) ccp_dm_free(&final_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (ilen > 0 && !in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) ccp_free_data(&dst, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (ilen > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) e_aad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) if (aes->aad_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) ccp_free_data(&aad, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) e_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) ccp_dm_free(&key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) struct ccp_aes_engine *aes = &cmd->u.aes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) struct ccp_dm_workarea key, ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) struct ccp_data src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) unsigned int dm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) bool in_place = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) if (!((aes->key_len == AES_KEYSIZE_128) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) (aes->key_len == AES_KEYSIZE_192) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) (aes->key_len == AES_KEYSIZE_256)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (((aes->mode == CCP_AES_MODE_ECB) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) (aes->mode == CCP_AES_MODE_CBC)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) (aes->src_len & (AES_BLOCK_SIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (!aes->key || !aes->src || !aes->dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) if (aes->mode != CCP_AES_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) if (aes->iv_len != AES_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (!aes->iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) op.sb_ctx = cmd_q->sb_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) op.u.aes.type = aes->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) op.u.aes.mode = aes->mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) op.u.aes.action = aes->action;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /* All supported key sizes fit in a single (32-byte) SB entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * and must be in little endian format. Use the 256-bit byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * swap passthru option to convert from big endian to little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) ret = ccp_init_dm_workarea(&key, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) dm_offset = CCP_SB_BYTES - aes->key_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) /* The AES context fits in a single (32-byte) SB entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * must be in little endian format. Use the 256-bit byte swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * passthru option to convert from big endian to little endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) ret = ccp_init_dm_workarea(&ctx, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) if (aes->mode != CCP_AES_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /* Load the AES context - convert to LE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) switch (aes->mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) case CCP_AES_MODE_CFB: /* CFB128 only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) case CCP_AES_MODE_CTR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) op.u.aes.size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) /* Prepare the input and output data workareas. For in-place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * operations we need to set the dma direction to BIDIRECTIONAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * and copy the src workarea to the dst workarea.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) if (sg_virt(aes->src) == sg_virt(aes->dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) in_place = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (in_place) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) dst = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) AES_BLOCK_SIZE, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) /* Send data to the CCP AES engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (!src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) /* Since we don't retrieve the AES context in ECB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * mode we have to wait for the operation to complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * on the last piece of data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) if (aes->mode == CCP_AES_MODE_ECB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) ret = cmd_q->ccp->vdata->perform->aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) ccp_process_data(&src, &dst, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (aes->mode != CCP_AES_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) /* Retrieve the AES context - convert from LE to BE using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * 32-byte (256-bit) byteswapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) /* ...but we only need AES_BLOCK_SIZE bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (!in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) ccp_free_data(&dst, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) e_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) ccp_dm_free(&key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) struct ccp_xts_aes_engine *xts = &cmd->u.xts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) struct ccp_dm_workarea key, ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) struct ccp_data src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) unsigned int unit_size, dm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) bool in_place = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) unsigned int sb_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) enum ccp_aes_type aestype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) switch (xts->unit_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) case CCP_XTS_AES_UNIT_SIZE_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) unit_size = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) case CCP_XTS_AES_UNIT_SIZE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) unit_size = 512;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) case CCP_XTS_AES_UNIT_SIZE_1024:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) unit_size = 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) case CCP_XTS_AES_UNIT_SIZE_2048:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) unit_size = 2048;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) case CCP_XTS_AES_UNIT_SIZE_4096:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) unit_size = 4096;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) if (xts->key_len == AES_KEYSIZE_128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) aestype = CCP_AES_TYPE_128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) else if (xts->key_len == AES_KEYSIZE_256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) aestype = CCP_AES_TYPE_256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (xts->iv_len != AES_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) if (!xts->key || !xts->iv || !xts->src || !xts->dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) op.sb_ctx = cmd_q->sb_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) op.init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) op.u.xts.type = aestype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) op.u.xts.action = xts->action;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) op.u.xts.unit_size = xts->unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) /* A version 3 device only supports 128-bit keys, which fits into a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * single SB entry. A version 5 device uses a 512-bit vector, so two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * SB entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) sb_count = CCP_XTS_AES_KEY_SB_COUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) ret = ccp_init_dm_workarea(&key, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) sb_count * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) /* All supported key sizes must be in little endian format.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * Use the 256-bit byte swap passthru option to convert from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * big endian to little endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) /* Version 5 CCPs use a 512-bit space for the key: each portion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) * occupies 256 bits, or one entire slot, and is zero-padded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) unsigned int pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) dm_offset = CCP_SB_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) pad = dm_offset - xts->key_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) xts->key_len, xts->key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /* The AES context fits in a single (32-byte) SB entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * for XTS is already in little endian format so no byte swapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) ret = ccp_init_dm_workarea(&ctx, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) CCP_PASSTHRU_BYTESWAP_NOOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) /* Prepare the input and output data workareas. For in-place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * operations we need to set the dma direction to BIDIRECTIONAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * and copy the src workarea to the dst workarea.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (sg_virt(xts->src) == sg_virt(xts->dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) in_place = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) unit_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) if (in_place) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) dst = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) unit_size, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) /* Send data to the CCP AES engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) ccp_prepare_data(&src, &dst, &op, unit_size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) if (!src.sg_wa.bytes_left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) ccp_process_data(&src, &dst, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) /* Retrieve the AES context - convert from LE to BE using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * 32-byte (256-bit) byteswapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) /* ...but we only need AES_BLOCK_SIZE bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (!in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) ccp_free_data(&dst, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) e_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) ccp_dm_free(&key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) struct ccp_des3_engine *des3 = &cmd->u.des3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) struct ccp_dm_workarea key, ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) struct ccp_data src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) unsigned int dm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) unsigned int len_singlekey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) bool in_place = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) /* Error checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) if (!cmd_q->ccp->vdata->perform->des3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) if (des3->key_len != DES3_EDE_KEY_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) if (((des3->mode == CCP_DES3_MODE_ECB) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) (des3->mode == CCP_DES3_MODE_CBC)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (!des3->key || !des3->src || !des3->dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) if (des3->mode != CCP_DES3_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (!des3->iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) /* Zero out all the fields of the command desc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) /* Set up the Function field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) op.u.des3.type = des3->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) op.u.des3.mode = des3->mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) op.u.des3.action = des3->action;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) * All supported key sizes fit in a single (32-byte) KSB entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * (like AES) must be in little endian format. Use the 256-bit byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) * swap passthru option to convert from big endian to little endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) ret = ccp_init_dm_workarea(&key, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * The contents of the key triplet are in the reverse order of what
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * is required by the engine. Copy the 3 pieces individually to put
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * them where they belong.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) len_singlekey = des3->key_len / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) des3->key, 0, len_singlekey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) des3->key, len_singlekey, len_singlekey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) ret = ccp_set_dm_area(&key, dm_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) des3->key, 2 * len_singlekey, len_singlekey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) /* Copy the key to the SB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) * The DES3 context fits in a single (32-byte) KSB entry and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * must be in little endian format. Use the 256-bit byte swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * passthru option to convert from big endian to little endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (des3->mode != CCP_DES3_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) op.sb_ctx = cmd_q->sb_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) ret = ccp_init_dm_workarea(&ctx, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) goto e_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) /* Load the context into the LSB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) dm_offset = CCP_SB_BYTES - des3->iv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) des3->iv_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) * Prepare the input and output data workareas. For in-place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) * operations we need to set the dma direction to BIDIRECTIONAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) * and copy the src workarea to the dst workarea.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) if (sg_virt(des3->src) == sg_virt(des3->dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) in_place = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) DES3_EDE_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) dst = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) /* Send data to the CCP DES3 engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) if (!src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) /* Since we don't retrieve the context in ECB mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) * we have to wait for the operation to complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) * on the last piece of data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) op.soc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) ret = cmd_q->ccp->vdata->perform->des3(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) ccp_process_data(&src, &dst, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) if (des3->mode != CCP_DES3_MODE_ECB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) /* Retrieve the context and make BE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) DES3_EDE_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) if (!in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) ccp_free_data(&dst, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (des3->mode != CCP_DES3_MODE_ECB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) e_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) ccp_dm_free(&key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) struct ccp_sha_engine *sha = &cmd->u.sha;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) struct ccp_dm_workarea ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) struct ccp_data src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) unsigned int ioffset, ooffset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) unsigned int digest_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) int sb_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) const void *init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) u64 block_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) int ctx_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) if (sha->ctx_len < SHA1_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) block_size = SHA1_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (sha->ctx_len < SHA224_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) block_size = SHA224_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (sha->ctx_len < SHA256_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) block_size = SHA256_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) case CCP_SHA_TYPE_384:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) || sha->ctx_len < SHA384_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) block_size = SHA384_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) case CCP_SHA_TYPE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) || sha->ctx_len < SHA512_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) block_size = SHA512_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) if (!sha->ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) if (!sha->final && (sha->src_len & (block_size - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) /* The version 3 device can't handle zero-length input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) if (!sha->src_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) unsigned int digest_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) const u8 *sha_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) /* Not final, just return */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (!sha->final)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) /* CCP can't do a zero length sha operation so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * caller must buffer the data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) if (sha->msg_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) /* The CCP cannot perform zero-length sha operations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) * so the caller is required to buffer data for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) * final operation. However, a sha operation for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) * message with a total length of zero is valid so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) * known values are required to supply the result.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) sha_zero = sha1_zero_message_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) digest_len = SHA1_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) sha_zero = sha224_zero_message_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) digest_len = SHA224_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) sha_zero = sha256_zero_message_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) digest_len = SHA256_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) digest_len, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) /* Set variables used throughout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) digest_size = SHA1_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) init = (void *) ccp_sha1_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) ctx_size = SHA1_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) sb_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) ooffset = ioffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) digest_size = SHA224_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) init = (void *) ccp_sha224_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) ctx_size = SHA256_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) sb_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) ioffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) ooffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) digest_size = SHA256_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) init = (void *) ccp_sha256_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) ctx_size = SHA256_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) sb_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) ooffset = ioffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) case CCP_SHA_TYPE_384:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) digest_size = SHA384_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) init = (void *) ccp_sha384_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) ctx_size = SHA512_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) sb_count = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) ioffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) case CCP_SHA_TYPE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) digest_size = SHA512_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) init = (void *) ccp_sha512_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) ctx_size = SHA512_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) sb_count = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) ooffset = ioffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) /* For zero-length plaintext the src pointer is ignored;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) * otherwise both parts must be valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) if (sha->src_len && !sha->src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) op.u.sha.type = sha->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) op.u.sha.msg_bits = sha->msg_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) * SHA384/512 require 2 adjacent SB slots, with the right half in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) * first slot, and the left half in the second. Each portion must then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) * be in little endian format: use the 256-bit byte swap option.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) if (sha->first) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) memcpy(ctx.address + ioffset, init, ctx_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) case CCP_SHA_TYPE_384:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) case CCP_SHA_TYPE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) memcpy(ctx.address + ctx_size / 2, init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) ctx_size / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) memcpy(ctx.address, init + ctx_size / 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) ctx_size / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) /* Restore the context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) sb_count * CCP_SB_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) if (sha->src) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) /* Send data to the CCP SHA engine; block_size is set above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) block_size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) goto e_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) while (src.sg_wa.bytes_left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) ccp_prepare_data(&src, NULL, &op, block_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) if (sha->final && !src.sg_wa.bytes_left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) ret = cmd_q->ccp->vdata->perform->sha(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) ccp_process_data(&src, NULL, &op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) ret = cmd_q->ccp->vdata->perform->sha(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) /* Retrieve the SHA context - convert from LE to BE using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) * 32-byte (256-bit) byteswapping to BE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) CCP_PASSTHRU_BYTESWAP_256BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (sha->final) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) /* Finishing up, so get the digest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) ccp_get_dm_area(&ctx, ooffset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) sha->ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) case CCP_SHA_TYPE_384:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) case CCP_SHA_TYPE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) ccp_get_dm_area(&ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) sha->ctx, LSB_ITEM_SIZE - ooffset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) LSB_ITEM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) sha->ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) LSB_ITEM_SIZE - ooffset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) /* Stash the context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) sb_count * CCP_SB_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (sha->final && sha->opad) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) /* HMAC operation, recursively perform final SHA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) struct ccp_cmd hmac_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) struct scatterlist sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) u8 *hmac_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) if (sha->opad_len != block_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) if (!hmac_buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) sg_init_one(&sg, hmac_buf, block_size + digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) switch (sha->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) case CCP_SHA_TYPE_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) case CCP_SHA_TYPE_224:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) case CCP_SHA_TYPE_256:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) memcpy(hmac_buf + block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) ctx.address + ooffset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) digest_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) case CCP_SHA_TYPE_384:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) case CCP_SHA_TYPE_512:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) memcpy(hmac_buf + block_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) ctx.address + LSB_ITEM_SIZE + ooffset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) LSB_ITEM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) memcpy(hmac_buf + block_size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) (LSB_ITEM_SIZE - ooffset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) ctx.address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) LSB_ITEM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) kfree(hmac_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) goto e_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) memset(&hmac_cmd, 0, sizeof(hmac_cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) hmac_cmd.engine = CCP_ENGINE_SHA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) hmac_cmd.u.sha.type = sha->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) hmac_cmd.u.sha.ctx = sha->ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) hmac_cmd.u.sha.ctx_len = sha->ctx_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) hmac_cmd.u.sha.src = &sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) hmac_cmd.u.sha.src_len = block_size + digest_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) hmac_cmd.u.sha.opad = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) hmac_cmd.u.sha.opad_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) hmac_cmd.u.sha.first = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) hmac_cmd.u.sha.final = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) cmd->engine_error = hmac_cmd.engine_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) kfree(hmac_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) e_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) if (sha->src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) e_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) ccp_dm_free(&ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) struct ccp_rsa_engine *rsa = &cmd->u.rsa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) struct ccp_dm_workarea exp, src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) unsigned int sb_count, i_len, o_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) /* Check against the maximum allowable size, in bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) /* The RSA modulus must precede the message being acted upon, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) * it must be copied to a DMA area where the message and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) * modulus can be concatenated. Therefore the input buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * length required is twice the output buffer length (which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) * Buffer sizes must be a multiple of 32 bytes; rounding up may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) * required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) o_len = 32 * ((rsa->key_size + 255) / 256);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) i_len = o_len * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) sb_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) /* sb_count is the number of storage block slots required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * for the modulus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) sb_count = o_len / CCP_SB_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) sb_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) if (!op.sb_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) /* A version 5 device allows a modulus size that will not fit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) * in the LSB, so the command will transfer it from memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) * Set the sb key to the default, even though it's not used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) /* The RSA exponent must be in little endian format. Reverse its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) * byte order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) goto e_sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) goto e_exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) /* Copy the exponent to the local storage block, using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) * as many 32-byte blocks as were allocated above. It's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) * already little endian, so no further change is required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) CCP_PASSTHRU_BYTESWAP_NOOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) goto e_exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) /* The exponent can be retrieved from memory via DMA. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) op.exp.u.dma.address = exp.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) op.exp.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) /* Concatenate the modulus and the message. Both the modulus and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) * the operands must be in little endian format. Since the input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) * is in big endian format it must be converted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) goto e_exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) /* Prepare the output area for the operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) op.src.u.dma.address = src.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) op.src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) op.src.u.dma.length = i_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) op.dst.u.dma.address = dst.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) op.dst.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) op.dst.u.dma.length = o_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) op.u.rsa.mod_size = rsa->key_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) op.u.rsa.input_len = i_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) ret = cmd_q->ccp->vdata->perform->rsa(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) ccp_dm_free(&dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) ccp_dm_free(&src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) e_exp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) ccp_dm_free(&exp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) e_sb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) if (sb_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) struct ccp_passthru_engine *pt = &cmd->u.passthru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) struct ccp_dm_workarea mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) struct ccp_data src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) bool in_place = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (!pt->src || !pt->dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) if (!pt->mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) /* Load the mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) ret = ccp_init_dm_workarea(&mask, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) CCP_PASSTHRU_SB_COUNT *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) CCP_SB_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) goto e_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) CCP_PASSTHRU_BYTESWAP_NOOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) goto e_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) /* Prepare the input and output data workareas. For in-place
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) * operations we need to set the dma direction to BIDIRECTIONAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) * and copy the src workarea to the dst workarea.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) if (sg_virt(pt->src) == sg_virt(pt->dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) in_place = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) CCP_PASSTHRU_MASKSIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) goto e_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) if (in_place) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) dst = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) /* Send data to the CCP Passthru engine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * Because the CCP engine works on a single source and destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * dma address at a time, each entry in the source scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) * (after the dma_map_sg call) must be less than or equal to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) * (remaining) length in the destination scatterlist entry and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) dst.sg_wa.sg_used = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) for (i = 1; i <= src.sg_wa.dma_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) if (!dst.sg_wa.sg ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) if (i == src.sg_wa.dma_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) op.src.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) op.src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) op.dst.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) op.dst.u.dma.offset = dst.sg_wa.sg_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) op.dst.u.dma.length = op.src.u.dma.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) ret = cmd_q->ccp->vdata->perform->passthru(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) dst.sg_wa.sg_used = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) src.sg_wa.sg = sg_next(src.sg_wa.sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) if (!in_place)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) ccp_free_data(&dst, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) ccp_free_data(&src, cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) e_mask:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) ccp_dm_free(&mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) struct ccp_dm_workarea mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) if (!pt->src_dma || !pt->dst_dma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) if (!pt->mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) /* Load the mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) op.sb_key = cmd_q->sb_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) mask.length = pt->mask_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) mask.dma.address = pt->mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) mask.dma.length = pt->mask_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) CCP_PASSTHRU_BYTESWAP_NOOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) /* Send data to the CCP Passthru engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) op.eom = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) op.src.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) op.src.u.dma.address = pt->src_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) op.src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) op.src.u.dma.length = pt->src_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) op.dst.type = CCP_MEMTYPE_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) op.dst.u.dma.address = pt->dst_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) op.dst.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) op.dst.u.dma.length = pt->src_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) ret = cmd_q->ccp->vdata->perform->passthru(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) struct ccp_ecc_engine *ecc = &cmd->u.ecc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) struct ccp_dm_workarea src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) u8 *save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) if (!ecc->u.mm.operand_1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) if (!ecc->u.mm.operand_2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) if (!ecc->u.mm.result ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) /* Concatenate the modulus and the operands. Both the modulus and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) * the operands must be in little endian format. Since the input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) * is in big endian format it must be converted and placed in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) * fixed length buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) /* Save the workarea address since it is updated in order to perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) * the concatenation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) save = src.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) /* Copy the ECC modulus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) /* Copy the first operand */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) ecc->u.mm.operand_1_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) /* Copy the second operand */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) ecc->u.mm.operand_2_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) /* Restore the workarea address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) src.address = save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) /* Prepare the output area for the operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) op.src.u.dma.address = src.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) op.src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) op.src.u.dma.length = src.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) op.dst.u.dma.address = dst.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) op.dst.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) op.dst.u.dma.length = dst.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) op.u.ecc.function = cmd->u.ecc.function;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) ret = cmd_q->ccp->vdata->perform->ecc(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) ecc->ecc_result = le16_to_cpup(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) /* Save the ECC result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) CCP_ECC_MODULUS_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) ccp_dm_free(&dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) ccp_dm_free(&src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) struct ccp_ecc_engine *ecc = &cmd->u.ecc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) struct ccp_dm_workarea src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) struct ccp_op op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) u8 *save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) if (!ecc->u.pm.point_1.x ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) !ecc->u.pm.point_1.y ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) if (!ecc->u.pm.point_2.x ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) !ecc->u.pm.point_2.y ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) if (!ecc->u.pm.domain_a ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) if (!ecc->u.pm.scalar ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) if (!ecc->u.pm.result.x ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) !ecc->u.pm.result.y ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) memset(&op, 0, sizeof(op));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) op.cmd_q = cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) /* Concatenate the modulus and the operands. Both the modulus and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) * the operands must be in little endian format. Since the input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) * is in big endian format it must be converted and placed in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) * fixed length buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) /* Save the workarea address since it is updated in order to perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) * the concatenation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) save = src.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) /* Copy the ECC modulus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) /* Copy the first point X and Y coordinate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) ecc->u.pm.point_1.x_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) ecc->u.pm.point_1.y_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) /* Set the first point Z coordinate to 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) *src.address = 0x01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) /* Copy the second point X and Y coordinate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) ecc->u.pm.point_2.x_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) ecc->u.pm.point_2.y_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) /* Set the second point Z coordinate to 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) *src.address = 0x01;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) /* Copy the Domain "a" parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) ecc->u.pm.domain_a_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) /* Copy the scalar value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) ret = ccp_reverse_set_dm_area(&src, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) ecc->u.pm.scalar, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) ecc->u.pm.scalar_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) src.address += CCP_ECC_OPERAND_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) /* Restore the workarea address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) src.address = save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) /* Prepare the output area for the operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) goto e_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) op.soc = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) op.src.u.dma.address = src.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) op.src.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) op.src.u.dma.length = src.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) op.dst.u.dma.address = dst.dma.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) op.dst.u.dma.offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) op.dst.u.dma.length = dst.length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) op.u.ecc.function = cmd->u.ecc.function;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) ret = cmd_q->ccp->vdata->perform->ecc(&op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) cmd->engine_error = cmd_q->cmd_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) ecc->ecc_result = le16_to_cpup(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) goto e_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) /* Save the workarea address since it is updated as we walk through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) * to copy the point math result
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) save = dst.address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) /* Save the ECC result X and Y coordinates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) CCP_ECC_MODULUS_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) dst.address += CCP_ECC_OUTPUT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) CCP_ECC_MODULUS_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) dst.address += CCP_ECC_OUTPUT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) /* Restore the workarea address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) dst.address = save;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) e_dst:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) ccp_dm_free(&dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) e_src:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) ccp_dm_free(&src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) struct ccp_ecc_engine *ecc = &cmd->u.ecc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) ecc->ecc_result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) if (!ecc->mod ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) switch (ecc->function) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) case CCP_ECC_FUNCTION_MMUL_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) case CCP_ECC_FUNCTION_MADD_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) case CCP_ECC_FUNCTION_MINV_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) return ccp_run_ecc_mm_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) case CCP_ECC_FUNCTION_PADD_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) case CCP_ECC_FUNCTION_PMUL_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) case CCP_ECC_FUNCTION_PDBL_384BIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) return ccp_run_ecc_pm_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) cmd->engine_error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) cmd_q->cmd_error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) cmd_q->int_rcvd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) switch (cmd->engine) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) case CCP_ENGINE_AES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) switch (cmd->u.aes.mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) case CCP_AES_MODE_CMAC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) case CCP_AES_MODE_GCM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) ret = ccp_run_aes_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) case CCP_ENGINE_XTS_AES_128:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) case CCP_ENGINE_DES3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) ret = ccp_run_des3_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) case CCP_ENGINE_SHA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) ret = ccp_run_sha_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) case CCP_ENGINE_RSA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) ret = ccp_run_rsa_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) case CCP_ENGINE_PASSTHRU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) ret = ccp_run_passthru_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) case CCP_ENGINE_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) ret = ccp_run_ecc_cmd(cmd_q, cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate