^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * caam - Freescale FSL CAAM support for ahash functions of crypto API
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright 2011 Freescale Semiconductor, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright 2018-2019 NXP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Based on caamalg.c crypto API driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * relationship of digest job descriptor or first job descriptor after init to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * shared descriptors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * --------------- ---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * | JobDesc #1 |-------------------->| ShareDesc |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * | *(packet 1) | | (hashKey) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * --------------- | (operation) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * ---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * relationship of subsequent job descriptors to shared descriptors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * --------------- ---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * | JobDesc #2 |-------------------->| ShareDesc |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * | *(packet 2) | |------------->| (hashKey) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * --------------- | |-------->| (operation) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * . | | | (load ctx2) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * . | | ---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * --------------- | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * | JobDesc #3 |------| |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * | *(packet 3) | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * --------------- |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * . |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * . |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * --------------- |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * | JobDesc #4 |------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * | *(packet 4) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * ---------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * The SharedDesc never changes for a connection unless rekeyed, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * each packet will likely be in a different place. So all we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * to know to process the packet is where the input is, where the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * output goes, and what context we want to process with. Context is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * in the SharedDesc, packet references in the JobDesc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * So, a job desc looks like:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * ---------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * | Header |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * | ShareDesc Pointer |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * | SEQ_OUT_PTR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * | (output buffer) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * | (output length) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * | SEQ_IN_PTR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * | (input buffer) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * | (input length) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * ---------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include "compat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "regs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include "intern.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include "desc_constr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include "jr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include "error.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include "sg_sw_sec4.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include "key_gen.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include "caamhash_desc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #include <crypto/engine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define CAAM_CRA_PRIORITY 3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* max hash key is max split key size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) CAAM_MAX_HASH_KEY_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /* caam context sizes for hashes: running digest + 8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define HASH_MSG_LEN 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static struct list_head hash_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* ahash per-session context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct caam_hash_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct crypto_engine_ctx enginectx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) u8 key[CAAM_MAX_HASH_KEY_SIZE] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) dma_addr_t sh_desc_update_dma ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) dma_addr_t sh_desc_update_first_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) dma_addr_t sh_desc_fin_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) dma_addr_t sh_desc_digest_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) enum dma_data_direction dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) enum dma_data_direction key_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct device *jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) int ctx_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct alginfo adata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* ahash state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct caam_hash_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) dma_addr_t buf_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) dma_addr_t ctx_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) int ctx_dma_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) int buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) int (*update)(struct ahash_request *req) ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) int (*final)(struct ahash_request *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) int (*finup)(struct ahash_request *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) void (*ahash_op_done)(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void *context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct caam_export_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u8 caam_ctx[MAX_CTX_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) int (*update)(struct ahash_request *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) int (*final)(struct ahash_request *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) int (*finup)(struct ahash_request *req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static inline bool is_cmac_aes(u32 algtype)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return (algtype & (OP_ALG_ALGSEL_MASK | OP_ALG_AAI_MASK)) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) (OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /* Common job descriptor seq in/out ptr routines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct caam_hash_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) int ctx_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) state->ctx_dma_len = ctx_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) ctx_len, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (dma_mapping_error(jrdev, state->ctx_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) dev_err(jrdev, "unable to map ctx\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) state->ctx_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* Map current buffer in state (if length > 0) and put it in link table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static inline int buf_map_to_sec4_sg(struct device *jrdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct sec4_sg_entry *sec4_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct caam_hash_state *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) int buflen = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (!buflen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) state->buf_dma = dma_map_single(jrdev, state->buf, buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (dma_mapping_error(jrdev, state->buf_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) dev_err(jrdev, "unable to map buf\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) state->buf_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* Map state->caam_ctx, and add it to link table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static inline int ctx_map_to_sec4_sg(struct device *jrdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct caam_hash_state *state, int ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct sec4_sg_entry *sec4_sg, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) state->ctx_dma_len = ctx_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (dma_mapping_error(jrdev, state->ctx_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) dev_err(jrdev, "unable to map ctx\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) state->ctx_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static int ahash_set_sh_desc(struct crypto_ahash *ahash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ctx->adata.key_virt = ctx->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* ahash_update shared descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) desc = ctx->sh_desc_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) ctx->ctx_len, true, ctrlpriv->era);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) /* ahash_update_first shared descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) desc = ctx->sh_desc_update_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) ctx->ctx_len, false, ctrlpriv->era);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* ahash_final shared descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) desc = ctx->sh_desc_fin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) ctx->ctx_len, true, ctrlpriv->era);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /* ahash_digest shared descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) desc = ctx->sh_desc_digest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) ctx->ctx_len, false, ctrlpriv->era);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /* shared descriptor for ahash_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) desc = ctx->sh_desc_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) ctx->ctx_len, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) print_hex_dump_debug("axcbc update shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /* shared descriptor for ahash_{final,finup} */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) desc = ctx->sh_desc_fin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) digestsize, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) print_hex_dump_debug("axcbc finup shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) /* key is immediate data for INIT and INITFINAL states */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ctx->adata.key_virt = ctx->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* shared descriptor for first invocation of ahash_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) desc = ctx->sh_desc_update_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* shared descriptor for ahash_digest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) desc = ctx->sh_desc_digest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) digestsize, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) print_hex_dump_debug("axcbc digest shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) static int acmac_set_sh_desc(struct crypto_ahash *ahash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /* shared descriptor for ahash_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) desc = ctx->sh_desc_update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_UPDATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) ctx->ctx_len, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) print_hex_dump_debug("acmac update shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) /* shared descriptor for ahash_{final,finup} */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) desc = ctx->sh_desc_fin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_FINALIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) digestsize, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) print_hex_dump_debug("acmac finup shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /* shared descriptor for first invocation of ahash_update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) desc = ctx->sh_desc_update_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /* shared descriptor for ahash_digest */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) desc = ctx->sh_desc_digest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) cnstr_shdsc_sk_hash(desc, &ctx->adata, OP_ALG_AS_INITFINAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) digestsize, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) desc_bytes(desc), ctx->dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) print_hex_dump_debug("acmac digest shdesc@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* Digest hash size if it is too large */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) u32 digestsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct split_key_result result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) dma_addr_t key_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (!desc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) dev_err(jrdev, "unable to allocate key input memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) init_job_desc(desc, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (dma_mapping_error(jrdev, key_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) dev_err(jrdev, "unable to map key memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) kfree(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /* Job descriptor to perform unkeyed hash on key_in */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) OP_ALG_AS_INITFINAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) append_seq_in_ptr(desc, key_dma, *keylen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) append_seq_out_ptr(desc, key_dma, digestsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) LDST_SRCDST_BYTE_CONTEXT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) result.err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) init_completion(&result.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (ret == -EINPROGRESS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) wait_for_completion(&result.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) ret = result.err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) DUMP_PREFIX_ADDRESS, 16, 4, key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) digestsize, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) *keylen = digestsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) kfree(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) static int ahash_setkey(struct crypto_ahash *ahash,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) const u8 *key, unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) u8 *hashed_key = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) dev_dbg(jrdev, "keylen %d\n", keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (keylen > blocksize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (!hashed_key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) goto bad_free_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) key = hashed_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * If DKP is supported, use it in the shared descriptor to generate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * the split key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (ctrlpriv->era >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) ctx->adata.key_inline = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) ctx->adata.keylen = keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) OP_ALG_ALGSEL_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) goto bad_free_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) memcpy(ctx->key, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * In case |user key| > |derived key|, using DKP<imm,imm>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * would result in invalid opcodes (last bytes of user key) in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * the resulting descriptor. Use DKP<ptr,imm> instead => both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * virtual and dma key addresses are needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (keylen > ctx->adata.keylen_pad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) dma_sync_single_for_device(ctx->jrdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) ctx->adata.key_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ctx->adata.keylen_pad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) keylen, CAAM_MAX_HASH_KEY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) goto bad_free_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) kfree(hashed_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return ahash_set_sh_desc(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) bad_free_key:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) kfree(hashed_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (keylen != AES_KEYSIZE_128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) memcpy(ctx->key, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) ctx->adata.keylen = keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return axcbc_set_sh_desc(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) err = aes_check_keylen(keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) /* key is immediate data for all cmac shared descriptors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ctx->adata.key_virt = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) ctx->adata.keylen = keylen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return acmac_set_sh_desc(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * ahash_edesc - s/w-extended ahash descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * @sec4_sg_dma: physical mapped address of h/w link table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * @src_nents: number of segments in input scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * @sec4_sg_bytes: length of dma mapped sec4_sg space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * @bklog: stored to determine if the request needs backlog
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * @hw_desc: the h/w job descriptor followed by any referenced link tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * @sec4_sg: h/w link table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) struct ahash_edesc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) dma_addr_t sec4_sg_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) int src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) bool bklog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) u32 hw_desc[DESC_JOB_IO_LEN_MAX / sizeof(u32)] ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct sec4_sg_entry sec4_sg[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static inline void ahash_unmap(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct ahash_edesc *edesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) struct ahash_request *req, int dst_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (edesc->src_nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (edesc->sec4_sg_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) dma_unmap_single(dev, edesc->sec4_sg_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) edesc->sec4_sg_bytes, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (state->buf_dma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) dma_unmap_single(dev, state->buf_dma, state->buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) state->buf_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) static inline void ahash_unmap_ctx(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct ahash_edesc *edesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct ahash_request *req, int dst_len, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (state->ctx_dma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) state->ctx_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) ahash_unmap(dev, edesc, req, dst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static inline void ahash_done_cpy(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) void *context, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) struct ahash_request *req = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) int ecode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) bool has_bklog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) edesc = state->edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) has_bklog = edesc->bklog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ecode = caam_jr_strstatus(jrdev, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ahash_unmap_ctx(jrdev, edesc, req, digestsize, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) memcpy(req->result, state->caam_ctx, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) ctx->ctx_len, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * If no backlog flag, the completion of the request is done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * by CAAM, not crypto engine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (!has_bklog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) req->base.complete(&req->base, ecode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) crypto_finalize_hash_request(jrp->engine, req, ecode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ahash_done_cpy(jrdev, desc, err, context, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) ahash_done_cpy(jrdev, desc, err, context, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) static inline void ahash_done_switch(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) void *context, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) struct ahash_request *req = context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) int ecode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) bool has_bklog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) edesc = state->edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) has_bklog = edesc->bklog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) ecode = caam_jr_strstatus(jrdev, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) scatterwalk_map_and_copy(state->buf, req->src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) req->nbytes - state->next_buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) state->next_buflen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) state->buflen = state->next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) DUMP_PREFIX_ADDRESS, 16, 4, state->buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) state->buflen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) ctx->ctx_len, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (req->result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) print_hex_dump_debug("result@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) DUMP_PREFIX_ADDRESS, 16, 4, req->result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) digestsize, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * If no backlog flag, the completion of the request is done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * by CAAM, not crypto engine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) if (!has_bklog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) req->base.complete(&req->base, ecode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) crypto_finalize_hash_request(jrp->engine, req, ecode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) ahash_done_switch(jrdev, desc, err, context, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) ahash_done_switch(jrdev, desc, err, context, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * Allocate an enhanced descriptor, which contains the hardware descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * and space for hardware scatter table containing sg_num entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) static struct ahash_edesc *ahash_edesc_alloc(struct ahash_request *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) int sg_num, u32 *sh_desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) dma_addr_t sh_desc_dma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) GFP_KERNEL : GFP_ATOMIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) state->edesc = edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) HDR_SHARE_DEFER | HDR_REVERSE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) return edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct ahash_edesc *edesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct ahash_request *req, int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) unsigned int first_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) unsigned int first_bytes, size_t to_hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) dma_addr_t src_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) u32 options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (nents > 1 || first_sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) struct sec4_sg_entry *sg = edesc->sec4_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) unsigned int sgsize = sizeof(*sg) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) pad_sg_nents(first_sg + nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (dma_mapping_error(ctx->jrdev, src_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) dev_err(ctx->jrdev, "unable to map S/G table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) edesc->sec4_sg_bytes = sgsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) edesc->sec4_sg_dma = src_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) options = LDST_SGF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) src_dma = sg_dma_address(req->src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) options = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) options);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) struct ahash_request *req = ahash_request_cast(areq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct caam_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) u32 *desc = state->edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) state->edesc->bklog = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (ret != -EINPROGRESS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) ahash_unmap(jrdev, state->edesc, req, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) kfree(state->edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) static int ahash_enqueue_req(struct device *jrdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) void (*cbk)(struct device *jrdev, u32 *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) u32 err, void *context),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) struct ahash_request *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) int dst_len, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) struct ahash_edesc *edesc = state->edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) u32 *desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) state->ahash_op_done = cbk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * Only the backlog request are sent to crypto-engine since the others
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * can be handled by CAAM, if free, especially since JR has up to 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * entries (more than the 10 entries from crypto-engine).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) ret = crypto_transfer_hash_request_to_engine(jrpriv->engine,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) ret = caam_jr_enqueue(jrdev, desc, cbk, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if ((ret != -EINPROGRESS) && (ret != -EBUSY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) ahash_unmap_ctx(jrdev, edesc, req, dst_len, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) /* submit update job descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) static int ahash_update_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) u8 *buf = state->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) int *buflen = &state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) int *next_buflen = &state->next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) int blocksize = crypto_ahash_blocksize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) int in_len = *buflen + req->nbytes, to_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) *next_buflen = in_len & (blocksize - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) to_hash = in_len - *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * For XCBC and CMAC, if to_hash is multiple of block size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * keep last block in internal buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if ((is_xcbc_aes(ctx->adata.algtype) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) (*next_buflen == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) *next_buflen = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) to_hash -= blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (to_hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) int pad_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) int src_len = req->nbytes - *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) src_nents = sg_nents_for_len(req->src, src_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) dev_err(jrdev, "unable to DMA map source\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * allocate space for base edesc and hw desc commands,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * link tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) edesc = ahash_edesc_alloc(req, pad_nents, ctx->sh_desc_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) ctx->sh_desc_update_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) edesc->sec4_sg_bytes = sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) edesc->sec4_sg, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (mapped_nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) sg_to_sec4_sg_last(req->src, src_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) edesc->sec4_sg + sec4_sg_src_index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) sec4_sg_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) dev_err(jrdev, "unable to map S/G table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) to_hash, LDST_SGF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) ret = ahash_enqueue_req(jrdev, ahash_done_bi, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) ctx->ctx_len, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) } else if (*next_buflen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) req->nbytes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) *buflen = *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) DUMP_PREFIX_ADDRESS, 16, 4, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) *buflen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) unmap_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) static int ahash_final_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) int buflen = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) int sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) sizeof(struct sec4_sg_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) /* allocate space for base edesc and hw desc commands, link tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) edesc = ahash_edesc_alloc(req, 4, ctx->sh_desc_fin,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) ctx->sh_desc_fin_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (!edesc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) edesc->sec4_sg_bytes = sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) edesc->sec4_sg, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) sec4_sg_bytes, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) dev_err(jrdev, "unable to map S/G table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) LDST_SGF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) digestsize, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) unmap_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) static int ahash_finup_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) int buflen = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) int sec4_sg_src_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) int src_nents, mapped_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) src_nents = sg_nents_for_len(req->src, req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) dev_err(jrdev, "unable to DMA map source\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) sec4_sg_src_index = 1 + (buflen ? 1 : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /* allocate space for base edesc and hw desc commands, link tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) ctx->sh_desc_fin, ctx->sh_desc_fin_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) edesc->sec4_sg, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) sec4_sg_src_index, ctx->ctx_len + buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) digestsize, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) unmap_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) static int ahash_digest(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) int src_nents, mapped_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) state->buf_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) src_nents = sg_nents_for_len(req->src, req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) dev_err(jrdev, "unable to map source for DMA\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) /* allocate space for base edesc and hw desc commands, link tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) edesc = ahash_edesc_alloc(req, mapped_nents > 1 ? mapped_nents : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) ctx->sh_desc_digest, ctx->sh_desc_digest_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) ahash_unmap(jrdev, edesc, req, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) ahash_unmap(jrdev, edesc, req, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) return ahash_enqueue_req(jrdev, ahash_done, req, digestsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) /* submit ahash final if it the first job descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) static int ahash_final_no_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) u8 *buf = state->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) int buflen = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) /* allocate space for base edesc and hw desc commands, link tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) edesc = ahash_edesc_alloc(req, 0, ctx->sh_desc_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) ctx->sh_desc_digest_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (!edesc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) if (buflen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) state->buf_dma = dma_map_single(jrdev, buf, buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (dma_mapping_error(jrdev, state->buf_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) dev_err(jrdev, "unable to map src\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return ahash_enqueue_req(jrdev, ahash_done, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) digestsize, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) ahash_unmap(jrdev, edesc, req, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) /* submit ahash update if it the first job descriptor after update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) static int ahash_update_no_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) u8 *buf = state->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) int *buflen = &state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) int *next_buflen = &state->next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) int blocksize = crypto_ahash_blocksize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) int in_len = *buflen + req->nbytes, to_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) int sec4_sg_bytes, src_nents, mapped_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) *next_buflen = in_len & (blocksize - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) to_hash = in_len - *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * For XCBC and CMAC, if to_hash is multiple of block size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * keep last block in internal buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if ((is_xcbc_aes(ctx->adata.algtype) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) (*next_buflen == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) *next_buflen = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) to_hash -= blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (to_hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) int pad_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) int src_len = req->nbytes - *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) src_nents = sg_nents_for_len(req->src, src_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) dev_err(jrdev, "unable to DMA map source\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) pad_nents = pad_sg_nents(1 + mapped_nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * allocate space for base edesc and hw desc commands,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * link tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) edesc = ahash_edesc_alloc(req, pad_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) ctx->sh_desc_update_first,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) ctx->sh_desc_update_first_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) edesc->sec4_sg_bytes = sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) sec4_sg_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) dev_err(jrdev, "unable to map S/G table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) ctx->ctx_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) if ((ret != -EINPROGRESS) && (ret != -EBUSY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) state->update = ahash_update_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) state->finup = ahash_finup_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) state->final = ahash_final_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) } else if (*next_buflen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) req->nbytes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) *buflen = *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) DUMP_PREFIX_ADDRESS, 16, 4, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) *buflen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) unmap_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) /* submit ahash finup if it the first job descriptor after update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) static int ahash_finup_no_ctx(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) int buflen = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) int digestsize = crypto_ahash_digestsize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) src_nents = sg_nents_for_len(req->src, req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) dev_err(jrdev, "unable to DMA map source\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) sec4_sg_src_index = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) sizeof(struct sec4_sg_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) /* allocate space for base edesc and hw desc commands, link tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) edesc = ahash_edesc_alloc(req, sec4_sg_src_index + mapped_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) ctx->sh_desc_digest, ctx->sh_desc_digest_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) edesc->sec4_sg_bytes = sec4_sg_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) req->nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) dev_err(jrdev, "unable to map S/G table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return ahash_enqueue_req(jrdev, ahash_done, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) digestsize, DMA_FROM_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) ahash_unmap(jrdev, edesc, req, digestsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) /* submit first update job descriptor after init */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) static int ahash_update_first(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) struct device *jrdev = ctx->jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) u8 *buf = state->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) int *buflen = &state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) int *next_buflen = &state->next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) int to_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) int blocksize = crypto_ahash_blocksize(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) u32 *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) int src_nents, mapped_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) struct ahash_edesc *edesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) *next_buflen = req->nbytes & (blocksize - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) to_hash = req->nbytes - *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * For XCBC and CMAC, if to_hash is multiple of block size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * keep last block in internal buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) if ((is_xcbc_aes(ctx->adata.algtype) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) is_cmac_aes(ctx->adata.algtype)) && to_hash >= blocksize &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) (*next_buflen == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) *next_buflen = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) to_hash -= blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) if (to_hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) src_nents = sg_nents_for_len(req->src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) req->nbytes - *next_buflen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) if (src_nents < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) dev_err(jrdev, "Invalid number of src SG.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) return src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if (src_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) if (!mapped_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) dev_err(jrdev, "unable to map source for DMA\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) mapped_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) * allocate space for base edesc and hw desc commands,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) * link tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) edesc = ahash_edesc_alloc(req, mapped_nents > 1 ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) mapped_nents : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) ctx->sh_desc_update_first,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) ctx->sh_desc_update_first_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) if (!edesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) edesc->src_nents = src_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) to_hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) desc = edesc->hw_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) goto unmap_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) DUMP_PREFIX_ADDRESS, 16, 4, desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) desc_bytes(desc), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) ctx->ctx_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if ((ret != -EINPROGRESS) && (ret != -EBUSY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) state->update = ahash_update_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) state->finup = ahash_finup_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) state->final = ahash_final_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) } else if (*next_buflen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) state->update = ahash_update_no_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) state->finup = ahash_finup_no_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) state->final = ahash_final_no_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) scatterwalk_map_and_copy(buf, req->src, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) req->nbytes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) *buflen = *next_buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) print_hex_dump_debug("buf@" __stringify(__LINE__)": ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) DUMP_PREFIX_ADDRESS, 16, 4, buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) *buflen, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) unmap_ctx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) kfree(edesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) static int ahash_finup_first(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) return ahash_digest(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) static int ahash_init(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) state->update = ahash_update_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) state->finup = ahash_finup_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) state->final = ahash_final_no_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) state->ctx_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) state->ctx_dma_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) state->buf_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) state->buflen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) state->next_buflen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) static int ahash_update(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) return state->update(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) static int ahash_finup(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) return state->finup(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) static int ahash_final(struct ahash_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) return state->final(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) static int ahash_export(struct ahash_request *req, void *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) struct caam_export_state *export = out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) u8 *buf = state->buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) int len = state->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) memcpy(export->buf, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) export->buflen = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) export->update = state->update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) export->final = state->final;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) export->finup = state->finup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) static int ahash_import(struct ahash_request *req, const void *in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) struct caam_hash_state *state = ahash_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) const struct caam_export_state *export = in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) memset(state, 0, sizeof(*state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) memcpy(state->buf, export->buf, export->buflen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) state->buflen = export->buflen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) state->update = export->update;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) state->final = export->final;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) state->finup = export->finup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) struct caam_hash_template {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) char name[CRYPTO_MAX_ALG_NAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) char driver_name[CRYPTO_MAX_ALG_NAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) char hmac_name[CRYPTO_MAX_ALG_NAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) unsigned int blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) struct ahash_alg template_ahash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) u32 alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) /* ahash descriptors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) static struct caam_hash_template driver_hash[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) .name = "sha1",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) .driver_name = "sha1-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) .hmac_name = "hmac(sha1)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) .hmac_driver_name = "hmac-sha1-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) .blocksize = SHA1_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) .digestsize = SHA1_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) .alg_type = OP_ALG_ALGSEL_SHA1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) .name = "sha224",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) .driver_name = "sha224-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) .hmac_name = "hmac(sha224)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) .hmac_driver_name = "hmac-sha224-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) .blocksize = SHA224_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) .digestsize = SHA224_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) .alg_type = OP_ALG_ALGSEL_SHA224,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) .name = "sha256",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) .driver_name = "sha256-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) .hmac_name = "hmac(sha256)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) .hmac_driver_name = "hmac-sha256-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) .blocksize = SHA256_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) .digestsize = SHA256_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) .alg_type = OP_ALG_ALGSEL_SHA256,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) .name = "sha384",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) .driver_name = "sha384-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) .hmac_name = "hmac(sha384)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) .hmac_driver_name = "hmac-sha384-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) .blocksize = SHA384_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) .digestsize = SHA384_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) .alg_type = OP_ALG_ALGSEL_SHA384,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) .name = "sha512",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) .driver_name = "sha512-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) .hmac_name = "hmac(sha512)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) .hmac_driver_name = "hmac-sha512-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) .blocksize = SHA512_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) .digestsize = SHA512_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) .alg_type = OP_ALG_ALGSEL_SHA512,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) .name = "md5",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) .driver_name = "md5-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) .hmac_name = "hmac(md5)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) .hmac_driver_name = "hmac-md5-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) .blocksize = MD5_BLOCK_WORDS * 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) .setkey = ahash_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) .digestsize = MD5_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) .alg_type = OP_ALG_ALGSEL_MD5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) .hmac_name = "xcbc(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) .hmac_driver_name = "xcbc-aes-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) .blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) .setkey = axcbc_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) .digestsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XCBC_MAC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) .hmac_name = "cmac(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) .hmac_driver_name = "cmac-aes-caam",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) .blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) .template_ahash = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) .init = ahash_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) .update = ahash_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) .final = ahash_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) .finup = ahash_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) .digest = ahash_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) .export = ahash_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) .import = ahash_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) .setkey = acmac_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) .halg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) .digestsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) .statesize = sizeof(struct caam_export_state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) .alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CMAC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) struct caam_hash_alg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) struct list_head entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) int alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) struct ahash_alg ahash_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) static int caam_hash_cra_init(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) struct crypto_alg *base = tfm->__crt_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) struct hash_alg_common *halg =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) container_of(base, struct hash_alg_common, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) struct ahash_alg *alg =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) container_of(halg, struct ahash_alg, halg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) struct caam_hash_alg *caam_hash =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) container_of(alg, struct caam_hash_alg, ahash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) HASH_MSG_LEN + SHA1_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) HASH_MSG_LEN + 32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) HASH_MSG_LEN + SHA256_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) HASH_MSG_LEN + 64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) HASH_MSG_LEN + SHA512_DIGEST_SIZE };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) const size_t sh_desc_update_offset = offsetof(struct caam_hash_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) sh_desc_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) dma_addr_t dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) struct caam_drv_private *priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) * Get a Job ring from Job Ring driver to ensure in-order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) * crypto request processing per tfm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) ctx->jrdev = caam_jr_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) if (IS_ERR(ctx->jrdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) pr_err("Job Ring Device allocation for transform failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) return PTR_ERR(ctx->jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) priv = dev_get_drvdata(ctx->jrdev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) if (is_xcbc_aes(caam_hash->alg_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) ctx->dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) ctx->key_dir = DMA_BIDIRECTIONAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) ctx->ctx_len = 48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) } else if (is_cmac_aes(caam_hash->alg_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) ctx->dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) ctx->key_dir = DMA_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) ctx->adata.algtype = OP_TYPE_CLASS1_ALG | caam_hash->alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) ctx->ctx_len = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) if (priv->era >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) ctx->dir = DMA_BIDIRECTIONAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) ctx->key_dir = alg->setkey ? DMA_TO_DEVICE : DMA_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) ctx->dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) ctx->key_dir = DMA_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) ctx->ctx_len = runninglen[(ctx->adata.algtype &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) OP_ALG_ALGSEL_SUBMASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) OP_ALG_ALGSEL_SHIFT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) if (ctx->key_dir != DMA_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) ARRAY_SIZE(ctx->key),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) ctx->key_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) dev_err(ctx->jrdev, "unable to map key\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) caam_jr_free(ctx->jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) offsetof(struct caam_hash_ctx, key) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) sh_desc_update_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) if (dma_mapping_error(ctx->jrdev, dma_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) dev_err(ctx->jrdev, "unable to map shared descriptors\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) if (ctx->key_dir != DMA_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) ARRAY_SIZE(ctx->key),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) ctx->key_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) caam_jr_free(ctx->jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) ctx->sh_desc_update_dma = dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) ctx->sh_desc_update_first_dma = dma_addr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) offsetof(struct caam_hash_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) sh_desc_update_first) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) sh_desc_update_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) sh_desc_fin) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) sh_desc_update_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) sh_desc_digest) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) sh_desc_update_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) ctx->enginectx.op.do_one_request = ahash_do_one_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) sizeof(struct caam_hash_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) * For keyed hash algorithms shared descriptors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) * will be created later in setkey() callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) return alg->setkey ? 0 : ahash_set_sh_desc(ahash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) static void caam_hash_cra_exit(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) offsetof(struct caam_hash_ctx, key) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) offsetof(struct caam_hash_ctx, sh_desc_update),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) if (ctx->key_dir != DMA_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) ARRAY_SIZE(ctx->key), ctx->key_dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) caam_jr_free(ctx->jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) void caam_algapi_hash_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) struct caam_hash_alg *t_alg, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) if (!hash_list.next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) crypto_unregister_ahash(&t_alg->ahash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) list_del(&t_alg->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) kfree(t_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) static struct caam_hash_alg *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) caam_hash_alloc(struct caam_hash_template *template,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) bool keyed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) struct caam_hash_alg *t_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) struct ahash_alg *halg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) struct crypto_alg *alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) if (!t_alg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) pr_err("failed to allocate t_alg\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) t_alg->ahash_alg = template->template_ahash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) halg = &t_alg->ahash_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) alg = &halg->halg.base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) if (keyed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) template->hmac_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) template->hmac_driver_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) template->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) template->driver_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) t_alg->ahash_alg.setkey = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) alg->cra_module = THIS_MODULE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) alg->cra_init = caam_hash_cra_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) alg->cra_exit = caam_hash_cra_exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) alg->cra_priority = CAAM_CRA_PRIORITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) alg->cra_blocksize = template->blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) alg->cra_alignmask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) t_alg->alg_type = template->alg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) return t_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) int caam_algapi_hash_init(struct device *ctrldev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) int i = 0, err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) unsigned int md_limit = SHA512_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) u32 md_inst, md_vid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) * Register crypto algorithms the device supports. First, identify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) * presence and attributes of MD block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) if (priv->era < 10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) md_vid = (rd_reg32(&priv->ctrl->perfmon.cha_id_ls) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) md_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) u32 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) md_inst = mdha & CHA_VER_NUM_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) * Skip registration of any hashing algorithms if MD block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) * is not present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) if (!md_inst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) /* Limit digest size based on LP256 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) if (md_vid == CHA_VER_VID_MD_LP256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) md_limit = SHA256_DIGEST_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) INIT_LIST_HEAD(&hash_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) /* register crypto algorithms the device supports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) struct caam_hash_alg *t_alg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) struct caam_hash_template *alg = driver_hash + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) /* If MD size is not supported by device, skip registration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) if (is_mdha(alg->alg_type) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) alg->template_ahash.halg.digestsize > md_limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) /* register hmac version */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) t_alg = caam_hash_alloc(alg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) if (IS_ERR(t_alg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) err = PTR_ERR(t_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) pr_warn("%s alg allocation failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) alg->hmac_driver_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) err = crypto_register_ahash(&t_alg->ahash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) pr_warn("%s alg registration failed: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) t_alg->ahash_alg.halg.base.cra_driver_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) kfree(t_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) list_add_tail(&t_alg->entry, &hash_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) if ((alg->alg_type & OP_ALG_ALGSEL_MASK) == OP_ALG_ALGSEL_AES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) /* register unkeyed version */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) t_alg = caam_hash_alloc(alg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) if (IS_ERR(t_alg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) err = PTR_ERR(t_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) pr_warn("%s alg allocation failed\n", alg->driver_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) err = crypto_register_ahash(&t_alg->ahash_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) pr_warn("%s alg registration failed: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) t_alg->ahash_alg.halg.base.cra_driver_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) kfree(t_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) list_add_tail(&t_alg->entry, &hash_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags