4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1) // SPDX-License-Identifier: GPL-2.0
03963caeb0dd7 (Gilad Ben-Yossef 2019-04-18 16:38:53 +0300 2) /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 3)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 4) #include <crypto/internal/aead.h>
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 5) #include <crypto/authenc.h>
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 6) #include <crypto/scatterwalk.h>
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 7) #include <linux/dmapool.h>
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 8) #include <linux/dma-mapping.h>
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 9)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 10) #include "cc_buffer_mgr.h"
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 11) #include "cc_lli_defs.h"
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 12) #include "cc_cipher.h"
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 13) #include "cc_hash.h"
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 14) #include "cc_aead.h"
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 15)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 16) union buffer_array_entry {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 17) struct scatterlist *sgl;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 18) dma_addr_t buffer_dma;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 19) };
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 20)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 21) struct buffer_array {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 22) unsigned int num_of_buffers;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 23) union buffer_array_entry entry[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 24) unsigned int offset[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 25) int nents[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 26) int total_data_len[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 27) bool is_last[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 28) u32 *mlli_nents[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 29) };
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 30)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 31) static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 32) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 33) switch (type) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 34) case CC_DMA_BUF_NULL:
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 35) return "BUF_NULL";
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 36) case CC_DMA_BUF_DLLI:
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 37) return "BUF_DLLI";
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 38) case CC_DMA_BUF_MLLI:
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 39) return "BUF_MLLI";
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 40) default:
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 41) return "BUF_INVALID";
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 42) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 43) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 44)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 45) /**
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 46) * cc_copy_mac() - Copy MAC to temporary location
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 47) *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 48) * @dev: device object
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 49) * @req: aead request object
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 50) * @dir: [IN] copy from/to sgl
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 51) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 52) static void cc_copy_mac(struct device *dev, struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 53) enum cc_sg_cpy_direct dir)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 54) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 55) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 56) u32 skip = req->assoclen + req->cryptlen;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 57)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 58) cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 59) (skip - areq_ctx->req_authsize), skip, dir);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 60) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 61)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 62) /**
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 63) * cc_get_sgl_nents() - Get scatterlist number of entries.
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 64) *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 65) * @dev: Device object
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 66) * @sg_list: SG list
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 67) * @nbytes: [IN] Total SGL data bytes.
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 68) * @lbytes: [OUT] Returns the amount of bytes at the last entry
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 69) *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 70) * Return:
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 71) * Number of entries in the scatterlist
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 72) */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 73) static unsigned int cc_get_sgl_nents(struct device *dev,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 74) struct scatterlist *sg_list,
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 75) unsigned int nbytes, u32 *lbytes)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 76) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 77) unsigned int nents = 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 78)
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 79) *lbytes = 0;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 80)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 81) while (nbytes && sg_list) {
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 82) nents++;
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 83) /* get the number of bytes in the last entry */
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 84) *lbytes = nbytes;
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 85) nbytes -= (sg_list->length > nbytes) ?
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 86) nbytes : sg_list->length;
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 87) sg_list = sg_next(sg_list);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 88) }
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 89)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 90) dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 91) return nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 92) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 93)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 94) /**
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 95) * cc_copy_sg_portion() - Copy scatter list data,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 96) * from to_skip to end, to dest and vice versa
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 97) *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 98) * @dev: Device object
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 99) * @dest: Buffer to copy to/from
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 100) * @sg: SG list
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 101) * @to_skip: Number of bytes to skip before copying
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 102) * @end: Offset of last byte to copy
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 103) * @direct: Transfer direction (true == from SG list to buffer, false == from
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100 104) * buffer to SG list)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 105) */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 106) void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 107) u32 to_skip, u32 end, enum cc_sg_cpy_direct direct)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 108) {
b7ec8530687a5 (Gilad Ben-Yossef 2019-04-18 16:38:52 +0300 109) u32 nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 110)
b7ec8530687a5 (Gilad Ben-Yossef 2019-04-18 16:38:52 +0300 111) nents = sg_nents_for_len(sg, end);
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100 112) sg_copy_buffer(sg, nents, dest, (end - to_skip + 1), to_skip,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 113) (direct == CC_SG_TO_BUF));
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 114) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 115)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 116) static int cc_render_buff_to_mlli(struct device *dev, dma_addr_t buff_dma,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 117) u32 buff_size, u32 *curr_nents,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 118) u32 **mlli_entry_pp)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 119) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 120) u32 *mlli_entry_p = *mlli_entry_pp;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 121) u32 new_nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 122)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 123) /* Verify there is no memory overflow*/
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 124) new_nents = (*curr_nents + buff_size / CC_MAX_MLLI_ENTRY_SIZE + 1);
32be4c5b0fd26 (Hadar Gat 2019-01-15 15:43:12 +0200 125) if (new_nents > MAX_NUM_OF_TOTAL_MLLI_ENTRIES) {
32be4c5b0fd26 (Hadar Gat 2019-01-15 15:43:12 +0200 126) dev_err(dev, "Too many mlli entries. current %d max %d\n",
32be4c5b0fd26 (Hadar Gat 2019-01-15 15:43:12 +0200 127) new_nents, MAX_NUM_OF_TOTAL_MLLI_ENTRIES);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 128) return -ENOMEM;
32be4c5b0fd26 (Hadar Gat 2019-01-15 15:43:12 +0200 129) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 130)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 131) /*handle buffer longer than 64 kbytes */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 132) while (buff_size > CC_MAX_MLLI_ENTRY_SIZE) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 133) cc_lli_set_addr(mlli_entry_p, buff_dma);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 134) cc_lli_set_size(mlli_entry_p, CC_MAX_MLLI_ENTRY_SIZE);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 135) dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n",
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 136) *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET],
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 137) mlli_entry_p[LLI_WORD1_OFFSET]);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 138) buff_dma += CC_MAX_MLLI_ENTRY_SIZE;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 139) buff_size -= CC_MAX_MLLI_ENTRY_SIZE;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 140) mlli_entry_p = mlli_entry_p + 2;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 141) (*curr_nents)++;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 142) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 143) /*Last entry */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 144) cc_lli_set_addr(mlli_entry_p, buff_dma);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 145) cc_lli_set_size(mlli_entry_p, buff_size);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 146) dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n",
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 147) *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET],
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 148) mlli_entry_p[LLI_WORD1_OFFSET]);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 149) mlli_entry_p = mlli_entry_p + 2;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 150) *mlli_entry_pp = mlli_entry_p;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 151) (*curr_nents)++;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 152) return 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 153) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 154)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 155) static int cc_render_sg_to_mlli(struct device *dev, struct scatterlist *sgl,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 156) u32 sgl_data_len, u32 sgl_offset,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 157) u32 *curr_nents, u32 **mlli_entry_pp)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 158) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 159) struct scatterlist *curr_sgl = sgl;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 160) u32 *mlli_entry_p = *mlli_entry_pp;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 161) s32 rc = 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 162)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 163) for ( ; (curr_sgl && sgl_data_len);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 164) curr_sgl = sg_next(curr_sgl)) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 165) u32 entry_data_len =
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 166) (sgl_data_len > sg_dma_len(curr_sgl) - sgl_offset) ?
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 167) sg_dma_len(curr_sgl) - sgl_offset :
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 168) sgl_data_len;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 169) sgl_data_len -= entry_data_len;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 170) rc = cc_render_buff_to_mlli(dev, sg_dma_address(curr_sgl) +
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 171) sgl_offset, entry_data_len,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 172) curr_nents, &mlli_entry_p);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 173) if (rc)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 174) return rc;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 175)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 176) sgl_offset = 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 177) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 178) *mlli_entry_pp = mlli_entry_p;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 179) return 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 180) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 181)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 182) static int cc_generate_mlli(struct device *dev, struct buffer_array *sg_data,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 183) struct mlli_params *mlli_params, gfp_t flags)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 184) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 185) u32 *mlli_p;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 186) u32 total_nents = 0, prev_total_nents = 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 187) int rc = 0, i;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 188)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 189) dev_dbg(dev, "NUM of SG's = %d\n", sg_data->num_of_buffers);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 190)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 191) /* Allocate memory from the pointed pool */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 192) mlli_params->mlli_virt_addr =
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 193) dma_pool_alloc(mlli_params->curr_pool, flags,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 194) &mlli_params->mlli_dma_addr);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 195) if (!mlli_params->mlli_virt_addr) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 196) dev_err(dev, "dma_pool_alloc() failed\n");
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 197) rc = -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 198) goto build_mlli_exit;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 199) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 200) /* Point to start of MLLI */
ba99b6f9bd59f (Geert Uytterhoeven 2020-02-11 19:19:03 +0100 201) mlli_p = mlli_params->mlli_virt_addr;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 202) /* go over all SG's and link it to one MLLI table */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 203) for (i = 0; i < sg_data->num_of_buffers; i++) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 204) union buffer_array_entry *entry = &sg_data->entry[i];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 205) u32 tot_len = sg_data->total_data_len[i];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 206) u32 offset = sg_data->offset[i];
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 207)
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 208) rc = cc_render_sg_to_mlli(dev, entry->sgl, tot_len, offset,
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 209) &total_nents, &mlli_p);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 210) if (rc)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 211) return rc;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 212)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 213) /* set last bit in the current table */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 214) if (sg_data->mlli_nents[i]) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 215) /*Calculate the current MLLI table length for the
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 216) *length field in the descriptor
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 217) */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 218) *sg_data->mlli_nents[i] +=
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 219) (total_nents - prev_total_nents);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 220) prev_total_nents = total_nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 221) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 222) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 223)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 224) /* Set MLLI size for the bypass operation */
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 225) mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 226)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 227) dev_dbg(dev, "MLLI params: virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n",
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 228) mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 229) mlli_params->mlli_len);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 230)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 231) build_mlli_exit:
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 232) return rc;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 233) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 234)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 235) static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 236) unsigned int nents, struct scatterlist *sgl,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 237) unsigned int data_len, unsigned int data_offset,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 238) bool is_last_table, u32 *mlli_nents)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 239) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 240) unsigned int index = sgl_data->num_of_buffers;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 241)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 242) dev_dbg(dev, "index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n",
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 243) index, nents, sgl, data_len, is_last_table);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 244) sgl_data->nents[index] = nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 245) sgl_data->entry[index].sgl = sgl;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 246) sgl_data->offset[index] = data_offset;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 247) sgl_data->total_data_len[index] = data_len;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 248) sgl_data->is_last[index] = is_last_table;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 249) sgl_data->mlli_nents[index] = mlli_nents;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 250) if (sgl_data->mlli_nents[index])
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 251) *sgl_data->mlli_nents[index] = 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 252) sgl_data->num_of_buffers++;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 253) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 254)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 255) static int cc_map_sg(struct device *dev, struct scatterlist *sg,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 256) unsigned int nbytes, int direction, u32 *nents,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 257) u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 258) {
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 259) int ret = 0;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 260)
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 261) *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes);
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 262) if (*nents > max_sg_nents) {
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 263) *nents = 0;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 264) dev_err(dev, "Too many fragments. current %d max %d\n",
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 265) *nents, max_sg_nents);
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 266) return -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 267) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 268)
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 269) ret = dma_map_sg(dev, sg, *nents, direction);
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 270) if (dma_mapping_error(dev, ret)) {
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 271) *nents = 0;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 272) dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret);
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 273) return -ENOMEM;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 274) }
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 275)
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 276) *mapped_nents = ret;
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 277)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 278) return 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 279) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 280)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 281) static int
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 282) cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 283) u8 *config_data, struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 284) unsigned int assoclen)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 285) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 286) dev_dbg(dev, " handle additional data config set to DLLI\n");
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 287) /* create sg for the current buffer */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 288) sg_init_one(&areq_ctx->ccm_adata_sg, config_data,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 289) AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 290) if (dma_map_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE) != 1) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 291) dev_err(dev, "dma_map_sg() config buffer failed\n");
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 292) return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 293) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 294) dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 295) &sg_dma_address(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 296) sg_page(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 297) sg_virt(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 298) areq_ctx->ccm_adata_sg.offset, areq_ctx->ccm_adata_sg.length);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 299) /* prepare for case of MLLI */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 300) if (assoclen > 0) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 301) cc_add_sg_entry(dev, sg_data, 1, &areq_ctx->ccm_adata_sg,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 302) (AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 303) 0, false, NULL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 304) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 305) return 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 306) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 307)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 308) static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 309) u8 *curr_buff, u32 curr_buff_cnt,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 310) struct buffer_array *sg_data)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 311) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 312) dev_dbg(dev, " handle curr buff %x set to DLLI\n", curr_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 313) /* create sg for the current buffer */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 314) sg_init_one(areq_ctx->buff_sg, curr_buff, curr_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 315) if (dma_map_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE) != 1) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 316) dev_err(dev, "dma_map_sg() src buffer failed\n");
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 317) return -ENOMEM;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 318) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 319) dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 320) &sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 321) sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 322) areq_ctx->buff_sg->length);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 323) areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 324) areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 325) areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 326) /* prepare for case of MLLI */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 327) cc_add_sg_entry(dev, sg_data, 1, areq_ctx->buff_sg, curr_buff_cnt, 0,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 328) false, NULL);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 329) return 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 330) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 331)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 332) void cc_unmap_cipher_request(struct device *dev, void *ctx,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 333) unsigned int ivsize, struct scatterlist *src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 334) struct scatterlist *dst)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 335) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 336) struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 337)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 338) if (req_ctx->gen_ctx.iv_dma_addr) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 339) dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n",
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 340) &req_ctx->gen_ctx.iv_dma_addr, ivsize);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 341) dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr,
6f17e00f77d8a (Gilad Ben-Yossef 2019-04-18 16:38:44 +0300 342) ivsize, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 343) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 344) /* Release pool */
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 345) if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI &&
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 346) req_ctx->mlli_params.mlli_virt_addr) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 347) dma_pool_free(req_ctx->mlli_params.curr_pool,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 348) req_ctx->mlli_params.mlli_virt_addr,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 349) req_ctx->mlli_params.mlli_dma_addr);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 350) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 351)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 352) dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 353) dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src));
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 354)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 355) if (src != dst) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 356) dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 357) dev_dbg(dev, "Unmapped req->dst=%pK\n", sg_virt(dst));
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 358) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 359) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 360)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 361) int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 362) unsigned int ivsize, unsigned int nbytes,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 363) void *info, struct scatterlist *src,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 364) struct scatterlist *dst, gfp_t flags)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 365) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 366) struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 367) struct mlli_params *mlli_params = &req_ctx->mlli_params;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 368) struct device *dev = drvdata_to_dev(drvdata);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 369) struct buffer_array sg_data;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 370) u32 dummy = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 371) int rc = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 372) u32 mapped_nents = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 373)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 374) req_ctx->dma_buf_type = CC_DMA_BUF_DLLI;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 375) mlli_params->curr_pool = NULL;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 376) sg_data.num_of_buffers = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 377)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 378) /* Map IV buffer */
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 379) if (ivsize) {
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100 380) dump_byte_array("iv", info, ivsize);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 381) req_ctx->gen_ctx.iv_dma_addr =
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100 382) dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 383) if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 384) dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 385) ivsize, info);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 386) return -ENOMEM;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 387) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 388) dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 389) ivsize, info, &req_ctx->gen_ctx.iv_dma_addr);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 390) } else {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 391) req_ctx->gen_ctx.iv_dma_addr = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 392) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 393)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 394) /* Map the src SGL */
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 395) rc = cc_map_sg(dev, src, nbytes, DMA_BIDIRECTIONAL, &req_ctx->in_nents,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 396) LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 397) if (rc)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 398) goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 399) if (mapped_nents > 1)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 400) req_ctx->dma_buf_type = CC_DMA_BUF_MLLI;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 401)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 402) if (src == dst) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 403) /* Handle inplace operation */
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 404) if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 405) req_ctx->out_nents = 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 406) cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 407) nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 408) &req_ctx->in_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 409) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 410) } else {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 411) /* Map the dst sg */
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 412) rc = cc_map_sg(dev, dst, nbytes, DMA_BIDIRECTIONAL,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 413) &req_ctx->out_nents, LLI_MAX_NUM_OF_DATA_ENTRIES,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 414) &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 415) if (rc)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 416) goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 417) if (mapped_nents > 1)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 418) req_ctx->dma_buf_type = CC_DMA_BUF_MLLI;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 419)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 420) if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 421) cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 422) nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 423) &req_ctx->in_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 424) cc_add_sg_entry(dev, &sg_data, req_ctx->out_nents, dst,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 425) nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 426) &req_ctx->out_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 427) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 428) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 429)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 430) if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 431) mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 432) rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 433) if (rc)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 434) goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 435) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 436)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 437) dev_dbg(dev, "areq_ctx->dma_buf_type = %s\n",
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 438) cc_dma_buf_type(req_ctx->dma_buf_type));
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 439)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 440) return 0;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 441)
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 442) cipher_exit:
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 443) cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 444) return rc;
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 445) }
63ee04c8b491e (Gilad Ben-Yossef 2018-01-22 09:27:01 +0000 446)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 447) void cc_unmap_aead_request(struct device *dev, struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 448) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 449) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 450) unsigned int hw_iv_size = areq_ctx->hw_iv_size;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 451) struct cc_drvdata *drvdata = dev_get_drvdata(dev);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 452)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 453) if (areq_ctx->mac_buf_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 454) dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 455) MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 456) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 457)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 458) if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 459) if (areq_ctx->hkey_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 460) dma_unmap_single(dev, areq_ctx->hkey_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 461) AES_BLOCK_SIZE, DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 462) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 463)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 464) if (areq_ctx->gcm_block_len_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 465) dma_unmap_single(dev, areq_ctx->gcm_block_len_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 466) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 467) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 468)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 469) if (areq_ctx->gcm_iv_inc1_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 470) dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 471) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 472) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 473)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 474) if (areq_ctx->gcm_iv_inc2_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 475) dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 476) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 477) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 478) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 479)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 480) if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 481) if (areq_ctx->ccm_iv0_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 482) dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 483) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 484) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 485)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 486) dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 487) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 488) if (areq_ctx->gen_ctx.iv_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 489) dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 490) hw_iv_size, DMA_BIDIRECTIONAL);
453431a54934d (Waiman Long 2020-08-06 23:18:13 -0700 491) kfree_sensitive(areq_ctx->gen_ctx.iv);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 492) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 493)
a49411959ea6d (Hadar Gat 2019-01-15 15:43:13 +0200 494) /* Release pool */
a49411959ea6d (Hadar Gat 2019-01-15 15:43:13 +0200 495) if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
a49411959ea6d (Hadar Gat 2019-01-15 15:43:13 +0200 496) areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) &&
a49411959ea6d (Hadar Gat 2019-01-15 15:43:13 +0200 497) (areq_ctx->mlli_params.mlli_virt_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 498) dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 499) &areq_ctx->mlli_params.mlli_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 500) areq_ctx->mlli_params.mlli_virt_addr);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 501) dma_pool_free(areq_ctx->mlli_params.curr_pool,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 502) areq_ctx->mlli_params.mlli_virt_addr,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 503) areq_ctx->mlli_params.mlli_dma_addr);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 504) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 505)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 506) dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 507) sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents,
da3cf67f1bcf2 (Gilad Ben-Yossef 2019-04-18 16:38:59 +0300 508) areq_ctx->assoclen, req->cryptlen);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 509)
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 510) dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents,
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 511) DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 512) if (req->src != req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 513) dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 514) sg_virt(req->dst));
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 515) dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 516) DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 517) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 518) if (drvdata->coherent &&
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 519) areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 520) req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 521) /* copy back mac from temporary location to deal with possible
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 522) * data memory overriding that caused by cache coherence
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 523) * problem.
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 524) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 525) cc_copy_mac(dev, req, CC_SG_FROM_BUF);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 526) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 527) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 528)
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 529) static bool cc_is_icv_frag(unsigned int sgl_nents, unsigned int authsize,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 530) u32 last_entry_data_size)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 531) {
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 532) return ((sgl_nents > 1) && (last_entry_data_size < authsize));
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 533) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 534)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 535) static int cc_aead_chain_iv(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 536) struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 537) struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 538) bool is_last, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 539) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 540) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 541) unsigned int hw_iv_size = areq_ctx->hw_iv_size;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 542) struct device *dev = drvdata_to_dev(drvdata);
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 543) gfp_t flags = cc_gfp_flags(&req->base);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 544) int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 545)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 546) if (!req->iv) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 547) areq_ctx->gen_ctx.iv_dma_addr = 0;
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 548) areq_ctx->gen_ctx.iv = NULL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 549) goto chain_iv_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 550) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 551)
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 552) areq_ctx->gen_ctx.iv = kmemdup(req->iv, hw_iv_size, flags);
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 553) if (!areq_ctx->gen_ctx.iv)
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 554) return -ENOMEM;
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 555)
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 556) areq_ctx->gen_ctx.iv_dma_addr =
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 557) dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size,
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 558) DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 559) if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 560) dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 561) hw_iv_size, req->iv);
453431a54934d (Waiman Long 2020-08-06 23:18:13 -0700 562) kfree_sensitive(areq_ctx->gen_ctx.iv);
e8662a6a5f8f7 (Gilad Ben-Yossef 2019-04-18 16:39:05 +0300 563) areq_ctx->gen_ctx.iv = NULL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 564) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 565) goto chain_iv_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 566) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 567)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 568) dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 569) hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 570)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 571) chain_iv_exit:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 572) return rc;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 573) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 574)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 575) static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 576) struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 577) struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 578) bool is_last, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 579) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 580) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 581) int rc = 0;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 582) int mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 583) struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 584)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 585) if (!sg_data) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 586) rc = -EINVAL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 587) goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 588) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 589)
da3cf67f1bcf2 (Gilad Ben-Yossef 2019-04-18 16:38:59 +0300 590) if (areq_ctx->assoclen == 0) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 591) areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 592) areq_ctx->assoc.nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 593) areq_ctx->assoc.mlli_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 594) dev_dbg(dev, "Chain assoc of length 0: buff_type=%s nents=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 595) cc_dma_buf_type(areq_ctx->assoc_buff_type),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 596) areq_ctx->assoc.nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 597) goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 598) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 599)
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 600) mapped_nents = sg_nents_for_len(req->src, areq_ctx->assoclen);
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 601) if (mapped_nents < 0)
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 602) return mapped_nents;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 603)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 604) if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 605) dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 606) mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 607) return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 608) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 609) areq_ctx->assoc.nents = mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 610)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 611) /* in CCM case we have additional entry for
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 612) * ccm header configurations
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 613) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 614) if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 615) if ((mapped_nents + 1) > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 616) dev_err(dev, "CCM case.Too many fragments. Current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 617) (areq_ctx->assoc.nents + 1),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 618) LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 619) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 620) goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 621) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 622) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 623)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 624) if (mapped_nents == 1 && areq_ctx->ccm_hdr_size == ccm_header_size_null)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 625) areq_ctx->assoc_buff_type = CC_DMA_BUF_DLLI;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 626) else
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 627) areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 628)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 629) if (do_chain || areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 630) dev_dbg(dev, "Chain assoc: buff_type=%s nents=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 631) cc_dma_buf_type(areq_ctx->assoc_buff_type),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 632) areq_ctx->assoc.nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 633) cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src,
da3cf67f1bcf2 (Gilad Ben-Yossef 2019-04-18 16:38:59 +0300 634) areq_ctx->assoclen, 0, is_last,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 635) &areq_ctx->assoc.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 636) areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 637) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 638)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 639) chain_assoc_exit:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 640) return rc;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 641) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 642)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 643) static void cc_prepare_aead_data_dlli(struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 644) u32 *src_last_bytes, u32 *dst_last_bytes)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 645) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 646) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 647) enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 648) unsigned int authsize = areq_ctx->req_authsize;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 649) struct scatterlist *sg;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 650) ssize_t offset;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 651)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 652) areq_ctx->is_icv_fragmented = false;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 653)
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 654) if ((req->src == req->dst) || direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 655) sg = areq_ctx->src_sgl;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 656) offset = *src_last_bytes - authsize;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 657) } else {
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 658) sg = areq_ctx->dst_sgl;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 659) offset = *dst_last_bytes - authsize;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 660) }
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 661)
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 662) areq_ctx->icv_dma_addr = sg_dma_address(sg) + offset;
c9877cbc83b6e (Gilad Ben-Yossef 2019-04-18 16:38:58 +0300 663) areq_ctx->icv_virt_addr = sg_virt(sg) + offset;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 664) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 665)
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 666) static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 667) struct aead_request *req,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 668) struct buffer_array *sg_data,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 669) u32 *src_last_bytes, u32 *dst_last_bytes,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 670) bool is_last_table)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 671) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 672) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 673) enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 674) unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 675) struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 676) struct scatterlist *sg;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 677)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 678) if (req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 679) /*INPLACE*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 680) cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 681) areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 682) areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 683) &areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 684)
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 685) areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 686) cc_is_icv_frag(areq_ctx->src.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 687) *src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 688)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 689) if (areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 690) /* Backup happens only when ICV is fragmented, ICV
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 691) * verification is made by CPU compare in order to
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 692) * simplify MAC verification upon request completion
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 693) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 694) if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 695) /* In coherent platforms (e.g. ACP)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 696) * already copying ICV for any
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 697) * INPLACE-DECRYPT operation, hence
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 698) * we must neglect this code.
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 699) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 700) if (!drvdata->coherent)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 701) cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 702)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 703) areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 704) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 705) areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 706) areq_ctx->icv_dma_addr =
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 707) areq_ctx->mac_buf_dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 708) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 709) } else { /* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 710) sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 711) /*Should hanlde if the sg is not contig.*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 712) areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 713) (*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 714) areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 715) (*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 716) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 717)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 718) } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 719) /*NON-INPLACE and DECRYPT*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 720) cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 721) areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 722) areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 723) &areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 724) cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 725) areq_ctx->dst_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 726) areq_ctx->dst_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 727) &areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 728)
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 729) areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 730) cc_is_icv_frag(areq_ctx->src.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 731) *src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 732) /* Backup happens only when ICV is fragmented, ICV
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 733)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 734) * verification is made by CPU compare in order to simplify
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 735) * MAC verification upon request completion
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 736) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 737) if (areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 738) cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 739) areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 740)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 741) } else { /* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 742) sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 743) /*Should hanlde if the sg is not contig.*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 744) areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 745) (*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 746) areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 747) (*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 748) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 749)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 750) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 751) /*NON-INPLACE and ENCRYPT*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 752) cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 753) areq_ctx->dst_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 754) areq_ctx->dst_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 755) &areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 756) cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 757) areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 758) areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 759) &areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 760)
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 761) areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 762) cc_is_icv_frag(areq_ctx->dst.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 763) *dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 764)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 765) if (!areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 766) sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 767) /* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 768) areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 769) (*dst_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 770) areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 771) (*dst_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 772) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 773) areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 774) areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 775) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 776) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 777) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 778)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 779) static int cc_aead_chain_data(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 780) struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 781) struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 782) bool is_last_table, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 783) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 784) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 785) struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 786) enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 787) unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 788) unsigned int src_last_bytes = 0, dst_last_bytes = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 789) int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 790) u32 src_mapped_nents = 0, dst_mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 791) u32 offset = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 792) /* non-inplace mode */
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 793) unsigned int size_for_map = req->assoclen + req->cryptlen;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 794) u32 sg_index = 0;
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 795) u32 size_to_skip = req->assoclen;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 796) struct scatterlist *sgl;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 797)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 798) offset = size_to_skip;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 799)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 800) if (!sg_data)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 801) return -EINVAL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 802)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 803) areq_ctx->src_sgl = req->src;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 804) areq_ctx->dst_sgl = req->dst;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 805)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 806) size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 807) authsize : 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 808) src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map,
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 809) &src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 810) sg_index = areq_ctx->src_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 811) //check where the data starts
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 812) while (src_mapped_nents && (sg_index <= size_to_skip)) {
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 813) src_mapped_nents--;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 814) offset -= areq_ctx->src_sgl->length;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 815) sgl = sg_next(areq_ctx->src_sgl);
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 816) if (!sgl)
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 817) break;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 818) areq_ctx->src_sgl = sgl;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 819) sg_index += areq_ctx->src_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 820) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 821) if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 822) dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 823) src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 824) return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 825) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 826)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 827) areq_ctx->src.nents = src_mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 828)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 829) areq_ctx->src_offset = offset;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 830)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 831) if (req->src != req->dst) {
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 832) size_for_map = req->assoclen + req->cryptlen;
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 833)
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 834) if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT)
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 835) size_for_map += authsize;
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 836) else
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 837) size_for_map -= authsize;
8962c6d2c2b8c (Gilad Ben-Yossef 2020-02-02 18:19:14 +0200 838)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 839) rc = cc_map_sg(dev, req->dst, size_for_map, DMA_BIDIRECTIONAL,
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 840) &areq_ctx->dst.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 841) LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 842) &dst_mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 843) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 844) goto chain_data_exit;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 845) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 846)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 847) dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map,
c4b22bf51b815 (Gilad Ben-Yossef 2019-04-18 16:38:48 +0300 848) &dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 849) sg_index = areq_ctx->dst_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 850) offset = size_to_skip;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 851)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 852) //check where the data starts
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 853) while (dst_mapped_nents && sg_index <= size_to_skip) {
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 854) dst_mapped_nents--;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 855) offset -= areq_ctx->dst_sgl->length;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 856) sgl = sg_next(areq_ctx->dst_sgl);
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 857) if (!sgl)
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 858) break;
c776f7d37b6bf (Gilad Ben-Yossef 2019-04-18 16:39:00 +0300 859) areq_ctx->dst_sgl = sgl;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 860) sg_index += areq_ctx->dst_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 861) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 862) if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 863) dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 864) dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 865) return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 866) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 867) areq_ctx->dst.nents = dst_mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 868) areq_ctx->dst_offset = offset;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 869) if (src_mapped_nents > 1 ||
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 870) dst_mapped_nents > 1 ||
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 871) do_chain) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 872) areq_ctx->data_buff_type = CC_DMA_BUF_MLLI;
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 873) cc_prepare_aead_data_mlli(drvdata, req, sg_data,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 874) &src_last_bytes, &dst_last_bytes,
6825cfd6d6f00 (Gilad Ben-Yossef 2019-04-18 16:38:57 +0300 875) is_last_table);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 876) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 877) areq_ctx->data_buff_type = CC_DMA_BUF_DLLI;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 878) cc_prepare_aead_data_dlli(req, &src_last_bytes,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 879) &dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 880) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 881)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 882) chain_data_exit:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 883) return rc;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 884) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 885)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 886) static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 887) struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 888) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 889) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 890) u32 curr_mlli_size = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 891)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 892) if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 893) areq_ctx->assoc.sram_addr = drvdata->mlli_sram_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 894) curr_mlli_size = areq_ctx->assoc.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 895) LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 896) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 897)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 898) if (areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 899) /*Inplace case dst nents equal to src nents*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 900) if (req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 901) areq_ctx->dst.mlli_nents = areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 902) areq_ctx->src.sram_addr = drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 903) curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 904) areq_ctx->dst.sram_addr = areq_ctx->src.sram_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 905) if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 906) areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 907) areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 908) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 909) if (areq_ctx->gen_ctx.op_type ==
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 910) DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 911) areq_ctx->src.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 912) drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 913) curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 914) areq_ctx->dst.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 915) areq_ctx->src.sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 916) areq_ctx->src.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 917) LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 918) if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 919) areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 920) areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 921) } else {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 922) areq_ctx->dst.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 923) drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 924) curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 925) areq_ctx->src.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 926) areq_ctx->dst.sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 927) areq_ctx->dst.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 928) LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 929) if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 930) areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 931) areq_ctx->dst.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 932) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 933) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 934) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 935) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 936)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 937) int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 938) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 939) struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 940) struct mlli_params *mlli_params = &areq_ctx->mlli_params;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 941) struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 942) struct buffer_array sg_data;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 943) unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 944) int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 945) dma_addr_t dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 946) u32 mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 947) u32 dummy = 0; /*used for the assoc data fragments */
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 948) u32 size_to_map;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 949) gfp_t flags = cc_gfp_flags(&req->base);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 950)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 951) mlli_params->curr_pool = NULL;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 952) sg_data.num_of_buffers = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 953)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 954) /* copy mac to a temporary location to deal with possible
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 955) * data memory overriding that caused by cache coherence problem.
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 956) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 957) if (drvdata->coherent &&
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 958) areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 959) req->src == req->dst)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 960) cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 961)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 962) /* cacluate the size for cipher remove ICV in decrypt*/
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 963) areq_ctx->cryptlen = (areq_ctx->gen_ctx.op_type ==
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 964) DRV_CRYPTO_DIRECTION_ENCRYPT) ?
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 965) req->cryptlen :
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 966) (req->cryptlen - authsize);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 967)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 968) dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 969) DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 970) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 971) dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 972) MAX_MAC_SIZE, areq_ctx->mac_buf);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 973) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 974) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 975) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 976) areq_ctx->mac_buf_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 977)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 978) if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 979) void *addr = areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 980)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 981) dma_addr = dma_map_single(dev, addr, AES_BLOCK_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 982) DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 983)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 984) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 985) dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 986) AES_BLOCK_SIZE, addr);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 987) areq_ctx->ccm_iv0_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 988) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 989) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 990) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 991) areq_ctx->ccm_iv0_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 992)
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 993) rc = cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config,
da3cf67f1bcf2 (Gilad Ben-Yossef 2019-04-18 16:38:59 +0300 994) &sg_data, areq_ctx->assoclen);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 995) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 996) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 997) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 998)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 999) if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1000) dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1001) DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1002) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1003) dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1004) AES_BLOCK_SIZE, areq_ctx->hkey);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1005) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1006) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1007) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1008) areq_ctx->hkey_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1009)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1010) dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1011) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1012) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1013) dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1014) AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1015) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1016) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1017) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1018) areq_ctx->gcm_block_len_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1019)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1020) dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc1,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1021) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1022)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1023) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1024) dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1025) AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1));
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1026) areq_ctx->gcm_iv_inc1_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1027) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1028) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1029) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1030) areq_ctx->gcm_iv_inc1_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1031)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1032) dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc2,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1033) AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1034)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1035) if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1036) dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1037) AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2));
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1038) areq_ctx->gcm_iv_inc2_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1039) rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1040) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1041) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1042) areq_ctx->gcm_iv_inc2_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1043) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1044)
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 1045) size_to_map = req->cryptlen + req->assoclen;
504e84abec7a6 (Gilad Ben-Yossef 2020-01-29 16:37:55 +0200 1046) /* If we do in-place encryption, we also need the auth tag */
504e84abec7a6 (Gilad Ben-Yossef 2020-01-29 16:37:55 +0200 1047) if ((areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) &&
504e84abec7a6 (Gilad Ben-Yossef 2020-01-29 16:37:55 +0200 1048) (req->src == req->dst)) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1049) size_to_map += authsize;
504e84abec7a6 (Gilad Ben-Yossef 2020-01-29 16:37:55 +0200 1050) }
0eae14a067c5b (Gilad Ben-Yossef 2020-03-08 17:57:09 +0200 1051)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1052) rc = cc_map_sg(dev, req->src, size_to_map, DMA_BIDIRECTIONAL,
ce0fc6db38dec (Gilad Ben-Yossef 2020-01-29 16:37:54 +0200 1053) &areq_ctx->src.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1054) (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES +
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1055) LLI_MAX_NUM_OF_DATA_ENTRIES),
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1056) &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1057) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1058) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1059)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1060) if (areq_ctx->is_single_pass) {
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1061) /*
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1062) * Create MLLI table for:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1063) * (1) Assoc. data
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1064) * (2) Src/Dst SGLs
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1065) * Note: IV is contg. buffer (not an SGL)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1066) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1067) rc = cc_aead_chain_assoc(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1068) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1069) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1070) rc = cc_aead_chain_iv(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1071) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1072) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1073) rc = cc_aead_chain_data(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1074) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1075) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1076) } else { /* DOUBLE-PASS flow */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1077) /*
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1078) * Prepare MLLI table(s) in this order:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1079) *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1080) * If ENCRYPT/DECRYPT (inplace):
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1081) * (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1082) * (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1083) * (3) MLLI for src/dst (inplace operation)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1084) *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1085) * If ENCRYPT (non-inplace)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1086) * (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1087) * (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1088) * (3) MLLI for dst
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1089) * (4) MLLI for src
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1090) *
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1091) * If DECRYPT (non-inplace)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1092) * (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1093) * (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1094) * (3) MLLI for src
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1095) * (4) MLLI for dst
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1096) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1097) rc = cc_aead_chain_assoc(drvdata, req, &sg_data, false, true);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1098) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1099) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1100) rc = cc_aead_chain_iv(drvdata, req, &sg_data, false, true);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1101) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1102) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1103) rc = cc_aead_chain_data(drvdata, req, &sg_data, true, true);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1104) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1105) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1106) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1107)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1108) /* Mlli support -start building the MLLI according to the above
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1109) * results
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1110) */
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1111) if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1112) areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1113) mlli_params->curr_pool = drvdata->mlli_buffs_pool;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1114) rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1115) if (rc)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1116) goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1117)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1118) cc_update_aead_mlli_nents(drvdata, req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1119) dev_dbg(dev, "assoc params mn %d\n",
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1120) areq_ctx->assoc.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1121) dev_dbg(dev, "src params mn %d\n", areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1122) dev_dbg(dev, "dst params mn %d\n", areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1123) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1124) return 0;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1125)
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1126) aead_map_failure:
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1127) cc_unmap_aead_request(dev, req);
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1128) return rc;
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1129) }
ff27e85a85bbd (Gilad Ben-Yossef 2018-01-22 09:27:03 +0000 1130)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1131) int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1132) struct scatterlist *src, unsigned int nbytes,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1133) bool do_update, gfp_t flags)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1134) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1135) struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1136) struct device *dev = drvdata_to_dev(drvdata);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1137) u8 *curr_buff = cc_hash_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1138) u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1139) struct mlli_params *mlli_params = &areq_ctx->mlli_params;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1140) struct buffer_array sg_data;
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1141) int rc = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1142) u32 dummy = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1143) u32 mapped_nents = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1144)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1145) dev_dbg(dev, "final params : curr_buff=%pK curr_buff_cnt=0x%X nbytes = 0x%X src=%pK curr_index=%u\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1146) curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1147) /* Init the type of the dma buffer */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1148) areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1149) mlli_params->curr_pool = NULL;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1150) sg_data.num_of_buffers = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1151) areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1152)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1153) if (nbytes == 0 && *curr_buff_cnt == 0) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1154) /* nothing to do */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1155) return 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1156) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1157)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1158) /* map the previous buffer */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1159) if (*curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1160) rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1161) &sg_data);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1162) if (rc)
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1163) return rc;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1164) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1165)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1166) if (src && nbytes > 0 && do_update) {
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1167) rc = cc_map_sg(dev, src, nbytes, DMA_TO_DEVICE,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1168) &areq_ctx->in_nents, LLI_MAX_NUM_OF_DATA_ENTRIES,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1169) &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1170) if (rc)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1171) goto unmap_curr_buff;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1172) if (src && mapped_nents == 1 &&
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1173) areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1174) memcpy(areq_ctx->buff_sg, src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1175) sizeof(struct scatterlist));
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1176) areq_ctx->buff_sg->length = nbytes;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1177) areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1178) areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1179) } else {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1180) areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1181) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1182) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1183)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1184) /*build mlli */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1185) if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1186) mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1187) /* add the src data to the sg_data */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1188) cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, nbytes,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1189) 0, true, &areq_ctx->mlli_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1190) rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1191) if (rc)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1192) goto fail_unmap_din;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1193) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1194) /* change the buffer index for the unmap function */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1195) areq_ctx->buff_index = (areq_ctx->buff_index ^ 1);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1196) dev_dbg(dev, "areq_ctx->data_dma_buf_type = %s\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1197) cc_dma_buf_type(areq_ctx->data_dma_buf_type));
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1198) return 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1199)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1200) fail_unmap_din:
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1201) dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1202)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1203) unmap_curr_buff:
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1204) if (*curr_buff_cnt)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1205) dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1206)
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1207) return rc;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1208) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1209)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1210) int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1211) struct scatterlist *src, unsigned int nbytes,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1212) unsigned int block_size, gfp_t flags)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1213) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1214) struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1215) struct device *dev = drvdata_to_dev(drvdata);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1216) u8 *curr_buff = cc_hash_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1217) u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1218) u8 *next_buff = cc_next_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1219) u32 *next_buff_cnt = cc_next_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1220) struct mlli_params *mlli_params = &areq_ctx->mlli_params;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1221) unsigned int update_data_len;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1222) u32 total_in_len = nbytes + *curr_buff_cnt;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1223) struct buffer_array sg_data;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1224) unsigned int swap_index = 0;
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1225) int rc = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1226) u32 dummy = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1227) u32 mapped_nents = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1228)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1229) dev_dbg(dev, " update params : curr_buff=%pK curr_buff_cnt=0x%X nbytes=0x%X src=%pK curr_index=%u\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1230) curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1231) /* Init the type of the dma buffer */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1232) areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1233) mlli_params->curr_pool = NULL;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1234) areq_ctx->curr_sg = NULL;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1235) sg_data.num_of_buffers = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1236) areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1237)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1238) if (total_in_len < block_size) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1239) dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1240) curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]);
b7ec8530687a5 (Gilad Ben-Yossef 2019-04-18 16:38:52 +0300 1241) areq_ctx->in_nents = sg_nents_for_len(src, nbytes);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1242) sg_copy_to_buffer(src, areq_ctx->in_nents,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1243) &curr_buff[*curr_buff_cnt], nbytes);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1244) *curr_buff_cnt += nbytes;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1245) return 1;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1246) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1247)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1248) /* Calculate the residue size*/
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1249) *next_buff_cnt = total_in_len & (block_size - 1);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1250) /* update data len */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1251) update_data_len = total_in_len - *next_buff_cnt;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1252)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1253) dev_dbg(dev, " temp length : *next_buff_cnt=0x%X update_data_len=0x%X\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1254) *next_buff_cnt, update_data_len);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1255)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1256) /* Copy the new residue to next buffer */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1257) if (*next_buff_cnt) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1258) dev_dbg(dev, " handle residue: next buff %pK skip data %u residue %u\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1259) next_buff, (update_data_len - *curr_buff_cnt),
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1260) *next_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1261) cc_copy_sg_portion(dev, next_buff, src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1262) (update_data_len - *curr_buff_cnt),
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1263) nbytes, CC_SG_TO_BUF);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1264) /* change the buffer index for next operation */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1265) swap_index = 1;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1266) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1267)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1268) if (*curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1269) rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1270) &sg_data);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1271) if (rc)
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1272) return rc;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1273) /* change the buffer index for next operation */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1274) swap_index = 1;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1275) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1276)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1277) if (update_data_len > *curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1278) rc = cc_map_sg(dev, src, (update_data_len - *curr_buff_cnt),
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1279) DMA_TO_DEVICE, &areq_ctx->in_nents,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1280) LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy,
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1281) &mapped_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1282) if (rc)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1283) goto unmap_curr_buff;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1284) if (mapped_nents == 1 &&
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1285) areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1286) /* only one entry in the SG and no previous data */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1287) memcpy(areq_ctx->buff_sg, src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1288) sizeof(struct scatterlist));
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1289) areq_ctx->buff_sg->length = update_data_len;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1290) areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1291) areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1292) } else {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1293) areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1294) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1295) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1296)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1297) if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1298) mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1299) /* add the src data to the sg_data */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1300) cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1301) (update_data_len - *curr_buff_cnt), 0, true,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1302) &areq_ctx->mlli_nents);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1303) rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1304) if (rc)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1305) goto fail_unmap_din;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1306) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1307) areq_ctx->buff_index = (areq_ctx->buff_index ^ swap_index);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1308)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1309) return 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1310)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1311) fail_unmap_din:
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1312) dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1313)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1314) unmap_curr_buff:
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1315) if (*curr_buff_cnt)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1316) dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1317)
ccba2f1112d48 (Hadar Gat 2019-01-15 15:43:11 +0200 1318) return rc;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1319) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1320)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1321) void cc_unmap_hash_request(struct device *dev, void *ctx,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1322) struct scatterlist *src, bool do_revert)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1323) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1324) struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1325) u32 *prev_len = cc_next_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1326)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1327) /*In case a pool was set, a table was
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1328) *allocated and should be released
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1329) */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1330) if (areq_ctx->mlli_params.curr_pool) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1331) dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1332) &areq_ctx->mlli_params.mlli_dma_addr,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1333) areq_ctx->mlli_params.mlli_virt_addr);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1334) dma_pool_free(areq_ctx->mlli_params.curr_pool,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1335) areq_ctx->mlli_params.mlli_virt_addr,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1336) areq_ctx->mlli_params.mlli_dma_addr);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1337) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1338)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1339) if (src && areq_ctx->in_nents) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1340) dev_dbg(dev, "Unmapped sg src: virt=%pK dma=%pad len=0x%X\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1341) sg_virt(src), &sg_dma_address(src), sg_dma_len(src));
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1342) dma_unmap_sg(dev, src,
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1343) areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1344) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1345)
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1346) if (*prev_len) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1347) dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%pK dma=%pad len 0x%X\n",
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1348) sg_virt(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1349) &sg_dma_address(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1350) sg_dma_len(areq_ctx->buff_sg));
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1351) dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1352) if (!do_revert) {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1353) /* clean the previous data length for update
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1354) * operation
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1355) */
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1356) *prev_len = 0;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1357) } else {
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1358) areq_ctx->buff_index ^= 1;
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1359) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1360) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1361) }
63893811b0fcb (Gilad Ben-Yossef 2018-01-22 09:27:02 +0000 1362)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1363) int cc_buffer_mgr_init(struct cc_drvdata *drvdata)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1364) {
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1365) struct device *dev = drvdata_to_dev(drvdata);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1366)
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1367) drvdata->mlli_buffs_pool =
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1368) dma_pool_create("dx_single_mlli_tables", dev,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1369) MAX_NUM_OF_TOTAL_MLLI_ENTRIES *
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1370) LLI_ENTRY_BYTE_SIZE,
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1371) MLLI_TABLE_MIN_ALIGNMENT, 0);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1372)
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1373) if (!drvdata->mlli_buffs_pool)
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1374) return -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1375)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1376) return 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1377) }
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1378)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1379) int cc_buffer_mgr_fini(struct cc_drvdata *drvdata)
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1380) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1381) dma_pool_destroy(drvdata->mlli_buffs_pool);
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1382) return 0;
4c3f97276e156 (Gilad Ben-Yossef 2018-01-22 09:27:00 +0000 1383) }