VisionFive2 Linux kernel

StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)

More than 9999 Commits   34 Branches   58 Tags
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    1) // SPDX-License-Identifier: GPL-2.0
03963caeb0dd7 (Gilad Ben-Yossef   2019-04-18 16:38:53 +0300    2) /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    3) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000    4) #include <crypto/internal/aead.h>
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    5) #include <crypto/authenc.h>
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    6) #include <crypto/scatterwalk.h>
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    7) #include <linux/dmapool.h>
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    8) #include <linux/dma-mapping.h>
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000    9) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   10) #include "cc_buffer_mgr.h"
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   11) #include "cc_lli_defs.h"
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000   12) #include "cc_cipher.h"
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000   13) #include "cc_hash.h"
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   14) #include "cc_aead.h"
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   15) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   16) union buffer_array_entry {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   17) 	struct scatterlist *sgl;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   18) 	dma_addr_t buffer_dma;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   19) };
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   20) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   21) struct buffer_array {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   22) 	unsigned int num_of_buffers;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   23) 	union buffer_array_entry entry[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   24) 	unsigned int offset[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   25) 	int nents[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   26) 	int total_data_len[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   27) 	bool is_last[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   28) 	u32 *mlli_nents[MAX_NUM_OF_BUFFERS_IN_MLLI];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   29) };
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   30) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   31) static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   32) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   33) 	switch (type) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   34) 	case CC_DMA_BUF_NULL:
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   35) 		return "BUF_NULL";
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   36) 	case CC_DMA_BUF_DLLI:
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   37) 		return "BUF_DLLI";
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   38) 	case CC_DMA_BUF_MLLI:
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   39) 		return "BUF_MLLI";
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   40) 	default:
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   41) 		return "BUF_INVALID";
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   42) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   43) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   44) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   45) /**
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   46)  * cc_copy_mac() - Copy MAC to temporary location
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   47)  *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   48)  * @dev: device object
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   49)  * @req: aead request object
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   50)  * @dir: [IN] copy from/to sgl
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   51)  */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   52) static void cc_copy_mac(struct device *dev, struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   53) 			enum cc_sg_cpy_direct dir)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   54) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   55) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200   56) 	u32 skip = req->assoclen + req->cryptlen;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   57) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   58) 	cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   59) 			   (skip - areq_ctx->req_authsize), skip, dir);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   60) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000   61) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   62) /**
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   63)  * cc_get_sgl_nents() - Get scatterlist number of entries.
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   64)  *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   65)  * @dev: Device object
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   66)  * @sg_list: SG list
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   67)  * @nbytes: [IN] Total SGL data bytes.
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   68)  * @lbytes: [OUT] Returns the amount of bytes at the last entry
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   69)  *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   70)  * Return:
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   71)  * Number of entries in the scatterlist
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   72)  */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   73) static unsigned int cc_get_sgl_nents(struct device *dev,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   74) 				     struct scatterlist *sg_list,
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   75) 				     unsigned int nbytes, u32 *lbytes)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   76) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   77) 	unsigned int nents = 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   78) 
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200   79) 	*lbytes = 0;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200   80) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   81) 	while (nbytes && sg_list) {
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   82) 		nents++;
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   83) 		/* get the number of bytes in the last entry */
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   84) 		*lbytes = nbytes;
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   85) 		nbytes -= (sg_list->length > nbytes) ?
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   86) 				nbytes : sg_list->length;
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300   87) 		sg_list = sg_next(sg_list);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   88) 	}
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200   89) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   90) 	dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   91) 	return nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   92) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   93) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   94) /**
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   95)  * cc_copy_sg_portion() - Copy scatter list data,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   96)  * from to_skip to end, to dest and vice versa
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000   97)  *
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   98)  * @dev: Device object
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100   99)  * @dest: Buffer to copy to/from
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100  100)  * @sg: SG list
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100  101)  * @to_skip: Number of bytes to skip before copying
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100  102)  * @end: Offset of last byte to copy
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100  103)  * @direct: Transfer direction (true == from SG list to buffer, false == from
ae02fcfeac13c (Geert Uytterhoeven 2020-02-11 19:19:19 +0100  104)  *          buffer to SG list)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  105)  */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  106) void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  107) 			u32 to_skip, u32 end, enum cc_sg_cpy_direct direct)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  108) {
b7ec8530687a5 (Gilad Ben-Yossef   2019-04-18 16:38:52 +0300  109) 	u32 nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  110) 
b7ec8530687a5 (Gilad Ben-Yossef   2019-04-18 16:38:52 +0300  111) 	nents = sg_nents_for_len(sg, end);
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100  112) 	sg_copy_buffer(sg, nents, dest, (end - to_skip + 1), to_skip,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  113) 		       (direct == CC_SG_TO_BUF));
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  114) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  115) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  116) static int cc_render_buff_to_mlli(struct device *dev, dma_addr_t buff_dma,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  117) 				  u32 buff_size, u32 *curr_nents,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  118) 				  u32 **mlli_entry_pp)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  119) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  120) 	u32 *mlli_entry_p = *mlli_entry_pp;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  121) 	u32 new_nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  122) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  123) 	/* Verify there is no memory overflow*/
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  124) 	new_nents = (*curr_nents + buff_size / CC_MAX_MLLI_ENTRY_SIZE + 1);
32be4c5b0fd26 (Hadar Gat          2019-01-15 15:43:12 +0200  125) 	if (new_nents > MAX_NUM_OF_TOTAL_MLLI_ENTRIES) {
32be4c5b0fd26 (Hadar Gat          2019-01-15 15:43:12 +0200  126) 		dev_err(dev, "Too many mlli entries. current %d max %d\n",
32be4c5b0fd26 (Hadar Gat          2019-01-15 15:43:12 +0200  127) 			new_nents, MAX_NUM_OF_TOTAL_MLLI_ENTRIES);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  128) 		return -ENOMEM;
32be4c5b0fd26 (Hadar Gat          2019-01-15 15:43:12 +0200  129) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  130) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  131) 	/*handle buffer longer than 64 kbytes */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  132) 	while (buff_size > CC_MAX_MLLI_ENTRY_SIZE) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  133) 		cc_lli_set_addr(mlli_entry_p, buff_dma);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  134) 		cc_lli_set_size(mlli_entry_p, CC_MAX_MLLI_ENTRY_SIZE);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  135) 		dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n",
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  136) 			*curr_nents, mlli_entry_p[LLI_WORD0_OFFSET],
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  137) 			mlli_entry_p[LLI_WORD1_OFFSET]);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  138) 		buff_dma += CC_MAX_MLLI_ENTRY_SIZE;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  139) 		buff_size -= CC_MAX_MLLI_ENTRY_SIZE;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  140) 		mlli_entry_p = mlli_entry_p + 2;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  141) 		(*curr_nents)++;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  142) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  143) 	/*Last entry */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  144) 	cc_lli_set_addr(mlli_entry_p, buff_dma);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  145) 	cc_lli_set_size(mlli_entry_p, buff_size);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  146) 	dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n",
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  147) 		*curr_nents, mlli_entry_p[LLI_WORD0_OFFSET],
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  148) 		mlli_entry_p[LLI_WORD1_OFFSET]);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  149) 	mlli_entry_p = mlli_entry_p + 2;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  150) 	*mlli_entry_pp = mlli_entry_p;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  151) 	(*curr_nents)++;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  152) 	return 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  153) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  154) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  155) static int cc_render_sg_to_mlli(struct device *dev, struct scatterlist *sgl,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  156) 				u32 sgl_data_len, u32 sgl_offset,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  157) 				u32 *curr_nents, u32 **mlli_entry_pp)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  158) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  159) 	struct scatterlist *curr_sgl = sgl;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  160) 	u32 *mlli_entry_p = *mlli_entry_pp;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  161) 	s32 rc = 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  162) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  163) 	for ( ; (curr_sgl && sgl_data_len);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  164) 	      curr_sgl = sg_next(curr_sgl)) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  165) 		u32 entry_data_len =
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  166) 			(sgl_data_len > sg_dma_len(curr_sgl) - sgl_offset) ?
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  167) 				sg_dma_len(curr_sgl) - sgl_offset :
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  168) 				sgl_data_len;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  169) 		sgl_data_len -= entry_data_len;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  170) 		rc = cc_render_buff_to_mlli(dev, sg_dma_address(curr_sgl) +
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  171) 					    sgl_offset, entry_data_len,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  172) 					    curr_nents, &mlli_entry_p);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  173) 		if (rc)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  174) 			return rc;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  175) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  176) 		sgl_offset = 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  177) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  178) 	*mlli_entry_pp = mlli_entry_p;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  179) 	return 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  180) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  181) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  182) static int cc_generate_mlli(struct device *dev, struct buffer_array *sg_data,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  183) 			    struct mlli_params *mlli_params, gfp_t flags)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  184) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  185) 	u32 *mlli_p;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  186) 	u32 total_nents = 0, prev_total_nents = 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  187) 	int rc = 0, i;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  188) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  189) 	dev_dbg(dev, "NUM of SG's = %d\n", sg_data->num_of_buffers);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  190) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  191) 	/* Allocate memory from the pointed pool */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  192) 	mlli_params->mlli_virt_addr =
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  193) 		dma_pool_alloc(mlli_params->curr_pool, flags,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  194) 			       &mlli_params->mlli_dma_addr);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  195) 	if (!mlli_params->mlli_virt_addr) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  196) 		dev_err(dev, "dma_pool_alloc() failed\n");
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  197) 		rc = -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  198) 		goto build_mlli_exit;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  199) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  200) 	/* Point to start of MLLI */
ba99b6f9bd59f (Geert Uytterhoeven 2020-02-11 19:19:03 +0100  201) 	mlli_p = mlli_params->mlli_virt_addr;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  202) 	/* go over all SG's and link it to one MLLI table */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  203) 	for (i = 0; i < sg_data->num_of_buffers; i++) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  204) 		union buffer_array_entry *entry = &sg_data->entry[i];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  205) 		u32 tot_len = sg_data->total_data_len[i];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  206) 		u32 offset = sg_data->offset[i];
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  207) 
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  208) 		rc = cc_render_sg_to_mlli(dev, entry->sgl, tot_len, offset,
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  209) 					  &total_nents, &mlli_p);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  210) 		if (rc)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  211) 			return rc;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  212) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  213) 		/* set last bit in the current table */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  214) 		if (sg_data->mlli_nents[i]) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  215) 			/*Calculate the current MLLI table length for the
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  216) 			 *length field in the descriptor
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  217) 			 */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  218) 			*sg_data->mlli_nents[i] +=
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  219) 				(total_nents - prev_total_nents);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  220) 			prev_total_nents = total_nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  221) 		}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  222) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  223) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  224) 	/* Set MLLI size for the bypass operation */
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  225) 	mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  226) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  227) 	dev_dbg(dev, "MLLI params: virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n",
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  228) 		mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  229) 		mlli_params->mlli_len);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  230) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  231) build_mlli_exit:
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  232) 	return rc;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  233) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  234) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  235) static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  236) 			    unsigned int nents, struct scatterlist *sgl,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  237) 			    unsigned int data_len, unsigned int data_offset,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  238) 			    bool is_last_table, u32 *mlli_nents)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  239) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  240) 	unsigned int index = sgl_data->num_of_buffers;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  241) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  242) 	dev_dbg(dev, "index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n",
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  243) 		index, nents, sgl, data_len, is_last_table);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  244) 	sgl_data->nents[index] = nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  245) 	sgl_data->entry[index].sgl = sgl;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  246) 	sgl_data->offset[index] = data_offset;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  247) 	sgl_data->total_data_len[index] = data_len;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  248) 	sgl_data->is_last[index] = is_last_table;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  249) 	sgl_data->mlli_nents[index] = mlli_nents;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  250) 	if (sgl_data->mlli_nents[index])
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  251) 		*sgl_data->mlli_nents[index] = 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  252) 	sgl_data->num_of_buffers++;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  253) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  254) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  255) static int cc_map_sg(struct device *dev, struct scatterlist *sg,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  256) 		     unsigned int nbytes, int direction, u32 *nents,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  257) 		     u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  258) {
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  259) 	int ret = 0;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  260) 
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  261) 	*nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes);
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  262) 	if (*nents > max_sg_nents) {
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  263) 		*nents = 0;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  264) 		dev_err(dev, "Too many fragments. current %d max %d\n",
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  265) 			*nents, max_sg_nents);
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  266) 		return -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  267) 	}
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  268) 
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  269) 	ret = dma_map_sg(dev, sg, *nents, direction);
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  270) 	if (dma_mapping_error(dev, ret)) {
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  271) 		*nents = 0;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  272) 		dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret);
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  273) 		return -ENOMEM;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  274) 	}
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  275) 
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  276) 	*mapped_nents = ret;
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  277) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  278) 	return 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  279) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000  280) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  281) static int
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  282) cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  283) 		     u8 *config_data, struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  284) 		     unsigned int assoclen)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  285) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  286) 	dev_dbg(dev, " handle additional data config set to DLLI\n");
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  287) 	/* create sg for the current buffer */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  288) 	sg_init_one(&areq_ctx->ccm_adata_sg, config_data,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  289) 		    AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  290) 	if (dma_map_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE) != 1) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  291) 		dev_err(dev, "dma_map_sg() config buffer failed\n");
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  292) 		return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  293) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  294) 	dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  295) 		&sg_dma_address(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  296) 		sg_page(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  297) 		sg_virt(&areq_ctx->ccm_adata_sg),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  298) 		areq_ctx->ccm_adata_sg.offset, areq_ctx->ccm_adata_sg.length);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  299) 	/* prepare for case of MLLI */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  300) 	if (assoclen > 0) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  301) 		cc_add_sg_entry(dev, sg_data, 1, &areq_ctx->ccm_adata_sg,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  302) 				(AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  303) 				0, false, NULL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  304) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  305) 	return 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  306) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  307) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  308) static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  309) 			   u8 *curr_buff, u32 curr_buff_cnt,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  310) 			   struct buffer_array *sg_data)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  311) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  312) 	dev_dbg(dev, " handle curr buff %x set to   DLLI\n", curr_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  313) 	/* create sg for the current buffer */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  314) 	sg_init_one(areq_ctx->buff_sg, curr_buff, curr_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  315) 	if (dma_map_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE) != 1) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  316) 		dev_err(dev, "dma_map_sg() src buffer failed\n");
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  317) 		return -ENOMEM;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  318) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  319) 	dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  320) 		&sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  321) 		sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  322) 		areq_ctx->buff_sg->length);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  323) 	areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  324) 	areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  325) 	areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  326) 	/* prepare for case of MLLI */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  327) 	cc_add_sg_entry(dev, sg_data, 1, areq_ctx->buff_sg, curr_buff_cnt, 0,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  328) 			false, NULL);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  329) 	return 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  330) }
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  331) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  332) void cc_unmap_cipher_request(struct device *dev, void *ctx,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  333) 				unsigned int ivsize, struct scatterlist *src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000  334) 				struct scatterlist *dst)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  335) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  336) 	struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  337) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  338) 	if (req_ctx->gen_ctx.iv_dma_addr) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  339) 		dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n",
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  340) 			&req_ctx->gen_ctx.iv_dma_addr, ivsize);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  341) 		dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr,
6f17e00f77d8a (Gilad Ben-Yossef   2019-04-18 16:38:44 +0300  342) 				 ivsize, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  343) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  344) 	/* Release pool */
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  345) 	if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI &&
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  346) 	    req_ctx->mlli_params.mlli_virt_addr) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  347) 		dma_pool_free(req_ctx->mlli_params.curr_pool,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  348) 			      req_ctx->mlli_params.mlli_virt_addr,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  349) 			      req_ctx->mlli_params.mlli_dma_addr);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  350) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  351) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  352) 	dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  353) 	dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src));
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  354) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  355) 	if (src != dst) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  356) 		dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  357) 		dev_dbg(dev, "Unmapped req->dst=%pK\n", sg_virt(dst));
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  358) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  359) }
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  360) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  361) int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  362) 			  unsigned int ivsize, unsigned int nbytes,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  363) 			  void *info, struct scatterlist *src,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  364) 			  struct scatterlist *dst, gfp_t flags)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  365) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  366) 	struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  367) 	struct mlli_params *mlli_params = &req_ctx->mlli_params;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  368) 	struct device *dev = drvdata_to_dev(drvdata);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  369) 	struct buffer_array sg_data;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  370) 	u32 dummy = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  371) 	int rc = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  372) 	u32 mapped_nents = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  373) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  374) 	req_ctx->dma_buf_type = CC_DMA_BUF_DLLI;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  375) 	mlli_params->curr_pool = NULL;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  376) 	sg_data.num_of_buffers = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  377) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  378) 	/* Map IV buffer */
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  379) 	if (ivsize) {
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100  380) 		dump_byte_array("iv", info, ivsize);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  381) 		req_ctx->gen_ctx.iv_dma_addr =
f4274eeca4763 (Geert Uytterhoeven 2020-02-11 19:18:58 +0100  382) 			dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  383) 		if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  384) 			dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  385) 				ivsize, info);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  386) 			return -ENOMEM;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  387) 		}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  388) 		dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  389) 			ivsize, info, &req_ctx->gen_ctx.iv_dma_addr);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  390) 	} else {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  391) 		req_ctx->gen_ctx.iv_dma_addr = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  392) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  393) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  394) 	/* Map the src SGL */
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  395) 	rc = cc_map_sg(dev, src, nbytes, DMA_BIDIRECTIONAL, &req_ctx->in_nents,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  396) 		       LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  397) 	if (rc)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  398) 		goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  399) 	if (mapped_nents > 1)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  400) 		req_ctx->dma_buf_type = CC_DMA_BUF_MLLI;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  401) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  402) 	if (src == dst) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  403) 		/* Handle inplace operation */
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  404) 		if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  405) 			req_ctx->out_nents = 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  406) 			cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  407) 					nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  408) 					&req_ctx->in_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  409) 		}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  410) 	} else {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  411) 		/* Map the dst sg */
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  412) 		rc = cc_map_sg(dev, dst, nbytes, DMA_BIDIRECTIONAL,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  413) 			       &req_ctx->out_nents, LLI_MAX_NUM_OF_DATA_ENTRIES,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  414) 			       &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  415) 		if (rc)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  416) 			goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  417) 		if (mapped_nents > 1)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  418) 			req_ctx->dma_buf_type = CC_DMA_BUF_MLLI;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  419) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  420) 		if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  421) 			cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  422) 					nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  423) 					&req_ctx->in_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  424) 			cc_add_sg_entry(dev, &sg_data, req_ctx->out_nents, dst,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  425) 					nbytes, 0, true,
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  426) 					&req_ctx->out_mlli_nents);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  427) 		}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  428) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  429) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  430) 	if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100  431) 		mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  432) 		rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  433) 		if (rc)
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  434) 			goto cipher_exit;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  435) 	}
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  436) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  437) 	dev_dbg(dev, "areq_ctx->dma_buf_type = %s\n",
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  438) 		cc_dma_buf_type(req_ctx->dma_buf_type));
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  439) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  440) 	return 0;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  441) 
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  442) cipher_exit:
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  443) 	cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst);
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  444) 	return rc;
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  445) }
63ee04c8b491e (Gilad Ben-Yossef   2018-01-22 09:27:01 +0000  446) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  447) void cc_unmap_aead_request(struct device *dev, struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  448) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  449) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  450) 	unsigned int hw_iv_size = areq_ctx->hw_iv_size;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  451) 	struct cc_drvdata *drvdata = dev_get_drvdata(dev);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  452) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  453) 	if (areq_ctx->mac_buf_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  454) 		dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  455) 				 MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  456) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  457) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  458) 	if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  459) 		if (areq_ctx->hkey_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  460) 			dma_unmap_single(dev, areq_ctx->hkey_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  461) 					 AES_BLOCK_SIZE, DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  462) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  463) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  464) 		if (areq_ctx->gcm_block_len_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  465) 			dma_unmap_single(dev, areq_ctx->gcm_block_len_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  466) 					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  467) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  468) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  469) 		if (areq_ctx->gcm_iv_inc1_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  470) 			dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  471) 					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  472) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  473) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  474) 		if (areq_ctx->gcm_iv_inc2_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  475) 			dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  476) 					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  477) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  478) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  479) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  480) 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  481) 		if (areq_ctx->ccm_iv0_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  482) 			dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  483) 					 AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  484) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  485) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  486) 		dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  487) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  488) 	if (areq_ctx->gen_ctx.iv_dma_addr) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  489) 		dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  490) 				 hw_iv_size, DMA_BIDIRECTIONAL);
453431a54934d (Waiman Long        2020-08-06 23:18:13 -0700  491) 		kfree_sensitive(areq_ctx->gen_ctx.iv);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  492) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  493) 
a49411959ea6d (Hadar Gat          2019-01-15 15:43:13 +0200  494) 	/* Release pool */
a49411959ea6d (Hadar Gat          2019-01-15 15:43:13 +0200  495) 	if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
a49411959ea6d (Hadar Gat          2019-01-15 15:43:13 +0200  496) 	     areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) &&
a49411959ea6d (Hadar Gat          2019-01-15 15:43:13 +0200  497) 	    (areq_ctx->mlli_params.mlli_virt_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  498) 		dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  499) 			&areq_ctx->mlli_params.mlli_dma_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  500) 			areq_ctx->mlli_params.mlli_virt_addr);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  501) 		dma_pool_free(areq_ctx->mlli_params.curr_pool,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  502) 			      areq_ctx->mlli_params.mlli_virt_addr,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  503) 			      areq_ctx->mlli_params.mlli_dma_addr);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  504) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  505) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  506) 	dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  507) 		sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents,
da3cf67f1bcf2 (Gilad Ben-Yossef   2019-04-18 16:38:59 +0300  508) 		areq_ctx->assoclen, req->cryptlen);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  509) 
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  510) 	dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents,
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  511) 		     DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  512) 	if (req->src != req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  513) 		dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  514) 			sg_virt(req->dst));
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  515) 		dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  516) 			     DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  517) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  518) 	if (drvdata->coherent &&
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  519) 	    areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  520) 	    req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  521) 		/* copy back mac from temporary location to deal with possible
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  522) 		 * data memory overriding that caused by cache coherence
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  523) 		 * problem.
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  524) 		 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  525) 		cc_copy_mac(dev, req, CC_SG_FROM_BUF);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  526) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  527) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  528) 
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  529) static bool cc_is_icv_frag(unsigned int sgl_nents, unsigned int authsize,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  530) 			   u32 last_entry_data_size)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  531) {
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  532) 	return ((sgl_nents > 1) && (last_entry_data_size < authsize));
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  533) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  534) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  535) static int cc_aead_chain_iv(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  536) 			    struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  537) 			    struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  538) 			    bool is_last, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  539) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  540) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  541) 	unsigned int hw_iv_size = areq_ctx->hw_iv_size;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  542) 	struct device *dev = drvdata_to_dev(drvdata);
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  543) 	gfp_t flags = cc_gfp_flags(&req->base);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  544) 	int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  545) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  546) 	if (!req->iv) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  547) 		areq_ctx->gen_ctx.iv_dma_addr = 0;
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  548) 		areq_ctx->gen_ctx.iv = NULL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  549) 		goto chain_iv_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  550) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  551) 
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  552) 	areq_ctx->gen_ctx.iv = kmemdup(req->iv, hw_iv_size, flags);
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  553) 	if (!areq_ctx->gen_ctx.iv)
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  554) 		return -ENOMEM;
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  555) 
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  556) 	areq_ctx->gen_ctx.iv_dma_addr =
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  557) 		dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size,
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  558) 			       DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  559) 	if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  560) 		dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  561) 			hw_iv_size, req->iv);
453431a54934d (Waiman Long        2020-08-06 23:18:13 -0700  562) 		kfree_sensitive(areq_ctx->gen_ctx.iv);
e8662a6a5f8f7 (Gilad Ben-Yossef   2019-04-18 16:39:05 +0300  563) 		areq_ctx->gen_ctx.iv = NULL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  564) 		rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  565) 		goto chain_iv_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  566) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  567) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  568) 	dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  569) 		hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  570) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  571) chain_iv_exit:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  572) 	return rc;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  573) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  574) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  575) static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  576) 			       struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  577) 			       struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  578) 			       bool is_last, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  579) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  580) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  581) 	int rc = 0;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  582) 	int mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  583) 	struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  584) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  585) 	if (!sg_data) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  586) 		rc = -EINVAL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  587) 		goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  588) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  589) 
da3cf67f1bcf2 (Gilad Ben-Yossef   2019-04-18 16:38:59 +0300  590) 	if (areq_ctx->assoclen == 0) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  591) 		areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  592) 		areq_ctx->assoc.nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  593) 		areq_ctx->assoc.mlli_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  594) 		dev_dbg(dev, "Chain assoc of length 0: buff_type=%s nents=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  595) 			cc_dma_buf_type(areq_ctx->assoc_buff_type),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  596) 			areq_ctx->assoc.nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  597) 		goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  598) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  599) 
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  600) 	mapped_nents = sg_nents_for_len(req->src, areq_ctx->assoclen);
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  601) 	if (mapped_nents < 0)
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  602) 		return mapped_nents;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  603) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  604) 	if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  605) 		dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  606) 			mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  607) 		return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  608) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  609) 	areq_ctx->assoc.nents = mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  610) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  611) 	/* in CCM case we have additional entry for
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  612) 	 * ccm header configurations
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  613) 	 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  614) 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  615) 		if ((mapped_nents + 1) > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  616) 			dev_err(dev, "CCM case.Too many fragments. Current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  617) 				(areq_ctx->assoc.nents + 1),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  618) 				LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  619) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  620) 			goto chain_assoc_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  621) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  622) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  623) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  624) 	if (mapped_nents == 1 && areq_ctx->ccm_hdr_size == ccm_header_size_null)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  625) 		areq_ctx->assoc_buff_type = CC_DMA_BUF_DLLI;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  626) 	else
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  627) 		areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  628) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  629) 	if (do_chain || areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  630) 		dev_dbg(dev, "Chain assoc: buff_type=%s nents=%u\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  631) 			cc_dma_buf_type(areq_ctx->assoc_buff_type),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  632) 			areq_ctx->assoc.nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  633) 		cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src,
da3cf67f1bcf2 (Gilad Ben-Yossef   2019-04-18 16:38:59 +0300  634) 				areq_ctx->assoclen, 0, is_last,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  635) 				&areq_ctx->assoc.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  636) 		areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  637) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  638) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  639) chain_assoc_exit:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  640) 	return rc;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  641) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  642) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  643) static void cc_prepare_aead_data_dlli(struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  644) 				      u32 *src_last_bytes, u32 *dst_last_bytes)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  645) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  646) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  647) 	enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  648) 	unsigned int authsize = areq_ctx->req_authsize;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  649) 	struct scatterlist *sg;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  650) 	ssize_t offset;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  651) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  652) 	areq_ctx->is_icv_fragmented = false;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  653) 
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  654) 	if ((req->src == req->dst) || direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  655) 		sg = areq_ctx->src_sgl;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  656) 		offset = *src_last_bytes - authsize;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  657) 	} else {
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  658) 		sg = areq_ctx->dst_sgl;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  659) 		offset = *dst_last_bytes - authsize;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  660) 	}
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  661) 
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  662) 	areq_ctx->icv_dma_addr = sg_dma_address(sg) + offset;
c9877cbc83b6e (Gilad Ben-Yossef   2019-04-18 16:38:58 +0300  663) 	areq_ctx->icv_virt_addr = sg_virt(sg) + offset;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  664) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  665) 
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  666) static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  667) 				      struct aead_request *req,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  668) 				      struct buffer_array *sg_data,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  669) 				      u32 *src_last_bytes, u32 *dst_last_bytes,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  670) 				      bool is_last_table)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  671) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  672) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  673) 	enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  674) 	unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  675) 	struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  676) 	struct scatterlist *sg;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  677) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  678) 	if (req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  679) 		/*INPLACE*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  680) 		cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  681) 				areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  682) 				areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  683) 				&areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  684) 
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  685) 		areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  686) 			cc_is_icv_frag(areq_ctx->src.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  687) 				       *src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  688) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  689) 		if (areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  690) 			/* Backup happens only when ICV is fragmented, ICV
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  691) 			 * verification is made by CPU compare in order to
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  692) 			 * simplify MAC verification upon request completion
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  693) 			 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  694) 			if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  695) 				/* In coherent platforms (e.g. ACP)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  696) 				 * already copying ICV for any
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  697) 				 * INPLACE-DECRYPT operation, hence
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  698) 				 * we must neglect this code.
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  699) 				 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  700) 				if (!drvdata->coherent)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  701) 					cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  702) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  703) 				areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  704) 			} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  705) 				areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  706) 				areq_ctx->icv_dma_addr =
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  707) 					areq_ctx->mac_buf_dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  708) 			}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  709) 		} else { /* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  710) 			sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  711) 			/*Should hanlde if the sg is not contig.*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  712) 			areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  713) 				(*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  714) 			areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  715) 				(*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  716) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  717) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  718) 	} else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  719) 		/*NON-INPLACE and DECRYPT*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  720) 		cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  721) 				areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  722) 				areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  723) 				&areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  724) 		cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  725) 				areq_ctx->dst_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  726) 				areq_ctx->dst_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  727) 				&areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  728) 
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  729) 		areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  730) 			cc_is_icv_frag(areq_ctx->src.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  731) 				       *src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  732) 		/* Backup happens only when ICV is fragmented, ICV
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  733) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  734) 		 * verification is made by CPU compare in order to simplify
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  735) 		 * MAC verification upon request completion
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  736) 		 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  737) 		if (areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  738) 			cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  739) 			areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  740) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  741) 		} else { /* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  742) 			sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  743) 			/*Should hanlde if the sg is not contig.*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  744) 			areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  745) 				(*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  746) 			areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  747) 				(*src_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  748) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  749) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  750) 	} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  751) 		/*NON-INPLACE and ENCRYPT*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  752) 		cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  753) 				areq_ctx->dst_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  754) 				areq_ctx->dst_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  755) 				&areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  756) 		cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  757) 				areq_ctx->src_sgl, areq_ctx->cryptlen,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  758) 				areq_ctx->src_offset, is_last_table,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  759) 				&areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  760) 
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  761) 		areq_ctx->is_icv_fragmented =
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  762) 			cc_is_icv_frag(areq_ctx->dst.nents, authsize,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  763) 				       *dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  764) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  765) 		if (!areq_ctx->is_icv_fragmented) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  766) 			sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1];
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  767) 			/* Contig. ICV */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  768) 			areq_ctx->icv_dma_addr = sg_dma_address(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  769) 				(*dst_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  770) 			areq_ctx->icv_virt_addr = sg_virt(sg) +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  771) 				(*dst_last_bytes - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  772) 		} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  773) 			areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  774) 			areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  775) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  776) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  777) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  778) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  779) static int cc_aead_chain_data(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  780) 			      struct aead_request *req,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  781) 			      struct buffer_array *sg_data,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  782) 			      bool is_last_table, bool do_chain)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  783) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  784) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  785) 	struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  786) 	enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  787) 	unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  788) 	unsigned int src_last_bytes = 0, dst_last_bytes = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  789) 	int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  790) 	u32 src_mapped_nents = 0, dst_mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  791) 	u32 offset = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  792) 	/* non-inplace mode */
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  793) 	unsigned int size_for_map = req->assoclen + req->cryptlen;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  794) 	u32 sg_index = 0;
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  795) 	u32 size_to_skip = req->assoclen;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  796) 	struct scatterlist *sgl;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  797) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  798) 	offset = size_to_skip;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  799) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  800) 	if (!sg_data)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  801) 		return -EINVAL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  802) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  803) 	areq_ctx->src_sgl = req->src;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  804) 	areq_ctx->dst_sgl = req->dst;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  805) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  806) 	size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  807) 			authsize : 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  808) 	src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map,
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300  809) 					    &src_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  810) 	sg_index = areq_ctx->src_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  811) 	//check where the data starts
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  812) 	while (src_mapped_nents && (sg_index <= size_to_skip)) {
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  813) 		src_mapped_nents--;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  814) 		offset -= areq_ctx->src_sgl->length;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  815) 		sgl = sg_next(areq_ctx->src_sgl);
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  816) 		if (!sgl)
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  817) 			break;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  818) 		areq_ctx->src_sgl = sgl;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  819) 		sg_index += areq_ctx->src_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  820) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  821) 	if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  822) 		dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  823) 			src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  824) 		return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  825) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  826) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  827) 	areq_ctx->src.nents = src_mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  828) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  829) 	areq_ctx->src_offset = offset;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  830) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  831) 	if (req->src != req->dst) {
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  832) 		size_for_map = req->assoclen + req->cryptlen;
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  833) 
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  834) 		if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT)
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  835) 			size_for_map += authsize;
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  836) 		else
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  837) 			size_for_map -= authsize;
8962c6d2c2b8c (Gilad Ben-Yossef   2020-02-02 18:19:14 +0200  838) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  839) 		rc = cc_map_sg(dev, req->dst, size_for_map, DMA_BIDIRECTIONAL,
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  840) 			       &areq_ctx->dst.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  841) 			       LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  842) 			       &dst_mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  843) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  844) 			goto chain_data_exit;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  845) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  846) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  847) 	dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map,
c4b22bf51b815 (Gilad Ben-Yossef   2019-04-18 16:38:48 +0300  848) 					    &dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  849) 	sg_index = areq_ctx->dst_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  850) 	offset = size_to_skip;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  851) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  852) 	//check where the data starts
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200  853) 	while (dst_mapped_nents && sg_index <= size_to_skip) {
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  854) 		dst_mapped_nents--;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  855) 		offset -= areq_ctx->dst_sgl->length;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  856) 		sgl = sg_next(areq_ctx->dst_sgl);
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  857) 		if (!sgl)
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  858) 			break;
c776f7d37b6bf (Gilad Ben-Yossef   2019-04-18 16:39:00 +0300  859) 		areq_ctx->dst_sgl = sgl;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  860) 		sg_index += areq_ctx->dst_sgl->length;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  861) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  862) 	if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  863) 		dev_err(dev, "Too many fragments. current %d max %d\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  864) 			dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  865) 		return -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  866) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  867) 	areq_ctx->dst.nents = dst_mapped_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  868) 	areq_ctx->dst_offset = offset;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  869) 	if (src_mapped_nents > 1 ||
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  870) 	    dst_mapped_nents  > 1 ||
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  871) 	    do_chain) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  872) 		areq_ctx->data_buff_type = CC_DMA_BUF_MLLI;
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  873) 		cc_prepare_aead_data_mlli(drvdata, req, sg_data,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  874) 					  &src_last_bytes, &dst_last_bytes,
6825cfd6d6f00 (Gilad Ben-Yossef   2019-04-18 16:38:57 +0300  875) 					  is_last_table);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  876) 	} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  877) 		areq_ctx->data_buff_type = CC_DMA_BUF_DLLI;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  878) 		cc_prepare_aead_data_dlli(req, &src_last_bytes,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  879) 					  &dst_last_bytes);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  880) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  881) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  882) chain_data_exit:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  883) 	return rc;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  884) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  885) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  886) static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  887) 				      struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  888) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  889) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  890) 	u32 curr_mlli_size = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  891) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  892) 	if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  893) 		areq_ctx->assoc.sram_addr = drvdata->mlli_sram_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  894) 		curr_mlli_size = areq_ctx->assoc.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  895) 						LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  896) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  897) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  898) 	if (areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  899) 		/*Inplace case dst nents equal to src nents*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  900) 		if (req->src == req->dst) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  901) 			areq_ctx->dst.mlli_nents = areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  902) 			areq_ctx->src.sram_addr = drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  903) 								curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  904) 			areq_ctx->dst.sram_addr = areq_ctx->src.sram_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  905) 			if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  906) 				areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  907) 					areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  908) 		} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  909) 			if (areq_ctx->gen_ctx.op_type ==
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  910) 					DRV_CRYPTO_DIRECTION_DECRYPT) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  911) 				areq_ctx->src.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  912) 						drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  913) 								curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  914) 				areq_ctx->dst.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  915) 						areq_ctx->src.sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  916) 						areq_ctx->src.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  917) 						LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  918) 				if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  919) 					areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  920) 						areq_ctx->src.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  921) 			} else {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  922) 				areq_ctx->dst.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  923) 						drvdata->mlli_sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  924) 								curr_mlli_size;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  925) 				areq_ctx->src.sram_addr =
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  926) 						areq_ctx->dst.sram_addr +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  927) 						areq_ctx->dst.mlli_nents *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  928) 						LLI_ENTRY_BYTE_SIZE;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  929) 				if (!areq_ctx->is_single_pass)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  930) 					areq_ctx->assoc.mlli_nents +=
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  931) 						areq_ctx->dst.mlli_nents;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  932) 			}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  933) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  934) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  935) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  936) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  937) int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  938) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  939) 	struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  940) 	struct mlli_params *mlli_params = &areq_ctx->mlli_params;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  941) 	struct device *dev = drvdata_to_dev(drvdata);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  942) 	struct buffer_array sg_data;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  943) 	unsigned int authsize = areq_ctx->req_authsize;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  944) 	int rc = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  945) 	dma_addr_t dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  946) 	u32 mapped_nents = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  947) 	u32 dummy = 0; /*used for the assoc data fragments */
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200  948) 	u32 size_to_map;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  949) 	gfp_t flags = cc_gfp_flags(&req->base);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  950) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  951) 	mlli_params->curr_pool = NULL;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  952) 	sg_data.num_of_buffers = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  953) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  954) 	/* copy mac to a temporary location to deal with possible
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  955) 	 * data memory overriding that caused by cache coherence problem.
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  956) 	 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  957) 	if (drvdata->coherent &&
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  958) 	    areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  959) 	    req->src == req->dst)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  960) 		cc_copy_mac(dev, req, CC_SG_TO_BUF);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  961) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  962) 	/* cacluate the size for cipher remove ICV in decrypt*/
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  963) 	areq_ctx->cryptlen = (areq_ctx->gen_ctx.op_type ==
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  964) 				 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  965) 				req->cryptlen :
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  966) 				(req->cryptlen - authsize);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  967) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  968) 	dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  969) 				  DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  970) 	if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  971) 		dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  972) 			MAX_MAC_SIZE, areq_ctx->mac_buf);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  973) 		rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  974) 		goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  975) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  976) 	areq_ctx->mac_buf_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  977) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  978) 	if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  979) 		void *addr = areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  980) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  981) 		dma_addr = dma_map_single(dev, addr, AES_BLOCK_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  982) 					  DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  983) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  984) 		if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  985) 			dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  986) 				AES_BLOCK_SIZE, addr);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  987) 			areq_ctx->ccm_iv0_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  988) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  989) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  990) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  991) 		areq_ctx->ccm_iv0_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  992) 
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  993) 		rc = cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config,
da3cf67f1bcf2 (Gilad Ben-Yossef   2019-04-18 16:38:59 +0300  994) 					  &sg_data, areq_ctx->assoclen);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200  995) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  996) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  997) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  998) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000  999) 	if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1000) 		dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1001) 					  DMA_BIDIRECTIONAL);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1002) 		if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1003) 			dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1004) 				AES_BLOCK_SIZE, areq_ctx->hkey);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1005) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1006) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1007) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1008) 		areq_ctx->hkey_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1009) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1010) 		dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1011) 					  AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1012) 		if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1013) 			dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1014) 				AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1015) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1016) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1017) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1018) 		areq_ctx->gcm_block_len_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1019) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1020) 		dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc1,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1021) 					  AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1022) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1023) 		if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1024) 			dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1025) 				AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1));
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1026) 			areq_ctx->gcm_iv_inc1_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1027) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1028) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1029) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1030) 		areq_ctx->gcm_iv_inc1_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1031) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1032) 		dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc2,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1033) 					  AES_BLOCK_SIZE, DMA_TO_DEVICE);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1034) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1035) 		if (dma_mapping_error(dev, dma_addr)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1036) 			dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1037) 				AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2));
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1038) 			areq_ctx->gcm_iv_inc2_dma_addr = 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1039) 			rc = -ENOMEM;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1040) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1041) 		}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1042) 		areq_ctx->gcm_iv_inc2_dma_addr = dma_addr;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1043) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1044) 
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200 1045) 	size_to_map = req->cryptlen + req->assoclen;
504e84abec7a6 (Gilad Ben-Yossef   2020-01-29 16:37:55 +0200 1046) 	/* If we do in-place encryption, we also need the auth tag */
504e84abec7a6 (Gilad Ben-Yossef   2020-01-29 16:37:55 +0200 1047) 	if ((areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) &&
504e84abec7a6 (Gilad Ben-Yossef   2020-01-29 16:37:55 +0200 1048) 	   (req->src == req->dst)) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1049) 		size_to_map += authsize;
504e84abec7a6 (Gilad Ben-Yossef   2020-01-29 16:37:55 +0200 1050) 	}
0eae14a067c5b (Gilad Ben-Yossef   2020-03-08 17:57:09 +0200 1051) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1052) 	rc = cc_map_sg(dev, req->src, size_to_map, DMA_BIDIRECTIONAL,
ce0fc6db38dec (Gilad Ben-Yossef   2020-01-29 16:37:54 +0200 1053) 		       &areq_ctx->src.mapped_nents,
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1054) 		       (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES +
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1055) 			LLI_MAX_NUM_OF_DATA_ENTRIES),
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1056) 		       &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1057) 	if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1058) 		goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1059) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1060) 	if (areq_ctx->is_single_pass) {
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1061) 		/*
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1062) 		 * Create MLLI table for:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1063) 		 *   (1) Assoc. data
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1064) 		 *   (2) Src/Dst SGLs
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1065) 		 *   Note: IV is contg. buffer (not an SGL)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1066) 		 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1067) 		rc = cc_aead_chain_assoc(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1068) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1069) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1070) 		rc = cc_aead_chain_iv(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1071) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1072) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1073) 		rc = cc_aead_chain_data(drvdata, req, &sg_data, true, false);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1074) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1075) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1076) 	} else { /* DOUBLE-PASS flow */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1077) 		/*
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1078) 		 * Prepare MLLI table(s) in this order:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1079) 		 *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1080) 		 * If ENCRYPT/DECRYPT (inplace):
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1081) 		 *   (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1082) 		 *   (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1083) 		 *   (3) MLLI for src/dst (inplace operation)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1084) 		 *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1085) 		 * If ENCRYPT (non-inplace)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1086) 		 *   (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1087) 		 *   (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1088) 		 *   (3) MLLI for dst
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1089) 		 *   (4) MLLI for src
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1090) 		 *
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1091) 		 * If DECRYPT (non-inplace)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1092) 		 *   (1) MLLI table for assoc
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1093) 		 *   (2) IV entry (chained right after end of assoc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1094) 		 *   (3) MLLI for src
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1095) 		 *   (4) MLLI for dst
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1096) 		 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1097) 		rc = cc_aead_chain_assoc(drvdata, req, &sg_data, false, true);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1098) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1099) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1100) 		rc = cc_aead_chain_iv(drvdata, req, &sg_data, false, true);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1101) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1102) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1103) 		rc = cc_aead_chain_data(drvdata, req, &sg_data, true, true);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1104) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1105) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1106) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1107) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1108) 	/* Mlli support -start building the MLLI according to the above
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1109) 	 * results
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1110) 	 */
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1111) 	if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1112) 	    areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1113) 		mlli_params->curr_pool = drvdata->mlli_buffs_pool;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1114) 		rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1115) 		if (rc)
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1116) 			goto aead_map_failure;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1117) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1118) 		cc_update_aead_mlli_nents(drvdata, req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1119) 		dev_dbg(dev, "assoc params mn %d\n",
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1120) 			areq_ctx->assoc.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1121) 		dev_dbg(dev, "src params mn %d\n", areq_ctx->src.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1122) 		dev_dbg(dev, "dst params mn %d\n", areq_ctx->dst.mlli_nents);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1123) 	}
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1124) 	return 0;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1125) 
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1126) aead_map_failure:
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1127) 	cc_unmap_aead_request(dev, req);
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1128) 	return rc;
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1129) }
ff27e85a85bbd (Gilad Ben-Yossef   2018-01-22 09:27:03 +0000 1130) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1131) int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1132) 			      struct scatterlist *src, unsigned int nbytes,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1133) 			      bool do_update, gfp_t flags)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1134) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1135) 	struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1136) 	struct device *dev = drvdata_to_dev(drvdata);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1137) 	u8 *curr_buff = cc_hash_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1138) 	u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1139) 	struct mlli_params *mlli_params = &areq_ctx->mlli_params;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1140) 	struct buffer_array sg_data;
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1141) 	int rc = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1142) 	u32 dummy = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1143) 	u32 mapped_nents = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1144) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1145) 	dev_dbg(dev, "final params : curr_buff=%pK curr_buff_cnt=0x%X nbytes = 0x%X src=%pK curr_index=%u\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1146) 		curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1147) 	/* Init the type of the dma buffer */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1148) 	areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1149) 	mlli_params->curr_pool = NULL;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1150) 	sg_data.num_of_buffers = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1151) 	areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1152) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1153) 	if (nbytes == 0 && *curr_buff_cnt == 0) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1154) 		/* nothing to do */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1155) 		return 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1156) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1157) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1158) 	/* map the previous buffer */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1159) 	if (*curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1160) 		rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1161) 				     &sg_data);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1162) 		if (rc)
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1163) 			return rc;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1164) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1165) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1166) 	if (src && nbytes > 0 && do_update) {
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1167) 		rc = cc_map_sg(dev, src, nbytes, DMA_TO_DEVICE,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1168) 			       &areq_ctx->in_nents, LLI_MAX_NUM_OF_DATA_ENTRIES,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1169) 			       &dummy, &mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1170) 		if (rc)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1171) 			goto unmap_curr_buff;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1172) 		if (src && mapped_nents == 1 &&
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1173) 		    areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1174) 			memcpy(areq_ctx->buff_sg, src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1175) 			       sizeof(struct scatterlist));
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1176) 			areq_ctx->buff_sg->length = nbytes;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1177) 			areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1178) 			areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1179) 		} else {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1180) 			areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1181) 		}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1182) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1183) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1184) 	/*build mlli */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1185) 	if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1186) 		mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1187) 		/* add the src data to the sg_data */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1188) 		cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, nbytes,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1189) 				0, true, &areq_ctx->mlli_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1190) 		rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1191) 		if (rc)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1192) 			goto fail_unmap_din;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1193) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1194) 	/* change the buffer index for the unmap function */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1195) 	areq_ctx->buff_index = (areq_ctx->buff_index ^ 1);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1196) 	dev_dbg(dev, "areq_ctx->data_dma_buf_type = %s\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1197) 		cc_dma_buf_type(areq_ctx->data_dma_buf_type));
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1198) 	return 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1199) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1200) fail_unmap_din:
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1201) 	dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1202) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1203) unmap_curr_buff:
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1204) 	if (*curr_buff_cnt)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1205) 		dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1206) 
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1207) 	return rc;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1208) }
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1209) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1210) int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1211) 			       struct scatterlist *src, unsigned int nbytes,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1212) 			       unsigned int block_size, gfp_t flags)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1213) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1214) 	struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1215) 	struct device *dev = drvdata_to_dev(drvdata);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1216) 	u8 *curr_buff = cc_hash_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1217) 	u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1218) 	u8 *next_buff = cc_next_buf(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1219) 	u32 *next_buff_cnt = cc_next_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1220) 	struct mlli_params *mlli_params = &areq_ctx->mlli_params;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1221) 	unsigned int update_data_len;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1222) 	u32 total_in_len = nbytes + *curr_buff_cnt;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1223) 	struct buffer_array sg_data;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1224) 	unsigned int swap_index = 0;
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1225) 	int rc = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1226) 	u32 dummy = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1227) 	u32 mapped_nents = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1228) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1229) 	dev_dbg(dev, " update params : curr_buff=%pK curr_buff_cnt=0x%X nbytes=0x%X src=%pK curr_index=%u\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1230) 		curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1231) 	/* Init the type of the dma buffer */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1232) 	areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1233) 	mlli_params->curr_pool = NULL;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1234) 	areq_ctx->curr_sg = NULL;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1235) 	sg_data.num_of_buffers = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1236) 	areq_ctx->in_nents = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1237) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1238) 	if (total_in_len < block_size) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1239) 		dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1240) 			curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]);
b7ec8530687a5 (Gilad Ben-Yossef   2019-04-18 16:38:52 +0300 1241) 		areq_ctx->in_nents = sg_nents_for_len(src, nbytes);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1242) 		sg_copy_to_buffer(src, areq_ctx->in_nents,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1243) 				  &curr_buff[*curr_buff_cnt], nbytes);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1244) 		*curr_buff_cnt += nbytes;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1245) 		return 1;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1246) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1247) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1248) 	/* Calculate the residue size*/
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1249) 	*next_buff_cnt = total_in_len & (block_size - 1);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1250) 	/* update data len */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1251) 	update_data_len = total_in_len - *next_buff_cnt;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1252) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1253) 	dev_dbg(dev, " temp length : *next_buff_cnt=0x%X update_data_len=0x%X\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1254) 		*next_buff_cnt, update_data_len);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1255) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1256) 	/* Copy the new residue to next buffer */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1257) 	if (*next_buff_cnt) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1258) 		dev_dbg(dev, " handle residue: next buff %pK skip data %u residue %u\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1259) 			next_buff, (update_data_len - *curr_buff_cnt),
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1260) 			*next_buff_cnt);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1261) 		cc_copy_sg_portion(dev, next_buff, src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1262) 				   (update_data_len - *curr_buff_cnt),
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1263) 				   nbytes, CC_SG_TO_BUF);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1264) 		/* change the buffer index for next operation */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1265) 		swap_index = 1;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1266) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1267) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1268) 	if (*curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1269) 		rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1270) 				     &sg_data);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1271) 		if (rc)
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1272) 			return rc;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1273) 		/* change the buffer index for next operation */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1274) 		swap_index = 1;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1275) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1276) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1277) 	if (update_data_len > *curr_buff_cnt) {
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1278) 		rc = cc_map_sg(dev, src, (update_data_len - *curr_buff_cnt),
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1279) 			       DMA_TO_DEVICE, &areq_ctx->in_nents,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1280) 			       LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy,
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1281) 			       &mapped_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1282) 		if (rc)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1283) 			goto unmap_curr_buff;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1284) 		if (mapped_nents == 1 &&
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1285) 		    areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1286) 			/* only one entry in the SG and no previous data */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1287) 			memcpy(areq_ctx->buff_sg, src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1288) 			       sizeof(struct scatterlist));
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1289) 			areq_ctx->buff_sg->length = update_data_len;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1290) 			areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1291) 			areq_ctx->curr_sg = areq_ctx->buff_sg;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1292) 		} else {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1293) 			areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1294) 		}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1295) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1296) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1297) 	if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1298) 		mlli_params->curr_pool = drvdata->mlli_buffs_pool;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1299) 		/* add the src data to the sg_data */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1300) 		cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1301) 				(update_data_len - *curr_buff_cnt), 0, true,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1302) 				&areq_ctx->mlli_nents);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1303) 		rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1304) 		if (rc)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1305) 			goto fail_unmap_din;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1306) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1307) 	areq_ctx->buff_index = (areq_ctx->buff_index ^ swap_index);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1308) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1309) 	return 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1310) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1311) fail_unmap_din:
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1312) 	dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1313) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1314) unmap_curr_buff:
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1315) 	if (*curr_buff_cnt)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1316) 		dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1317) 
ccba2f1112d48 (Hadar Gat          2019-01-15 15:43:11 +0200 1318) 	return rc;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1319) }
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1320) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1321) void cc_unmap_hash_request(struct device *dev, void *ctx,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1322) 			   struct scatterlist *src, bool do_revert)
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1323) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1324) 	struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1325) 	u32 *prev_len = cc_next_buf_cnt(areq_ctx);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1326) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1327) 	/*In case a pool was set, a table was
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1328) 	 *allocated and should be released
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1329) 	 */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1330) 	if (areq_ctx->mlli_params.curr_pool) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1331) 		dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1332) 			&areq_ctx->mlli_params.mlli_dma_addr,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1333) 			areq_ctx->mlli_params.mlli_virt_addr);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1334) 		dma_pool_free(areq_ctx->mlli_params.curr_pool,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1335) 			      areq_ctx->mlli_params.mlli_virt_addr,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1336) 			      areq_ctx->mlli_params.mlli_dma_addr);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1337) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1338) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1339) 	if (src && areq_ctx->in_nents) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1340) 		dev_dbg(dev, "Unmapped sg src: virt=%pK dma=%pad len=0x%X\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1341) 			sg_virt(src), &sg_dma_address(src), sg_dma_len(src));
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1342) 		dma_unmap_sg(dev, src,
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1343) 			     areq_ctx->in_nents, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1344) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1345) 
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1346) 	if (*prev_len) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1347) 		dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%pK dma=%pad len 0x%X\n",
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1348) 			sg_virt(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1349) 			&sg_dma_address(areq_ctx->buff_sg),
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1350) 			sg_dma_len(areq_ctx->buff_sg));
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1351) 		dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE);
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1352) 		if (!do_revert) {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1353) 			/* clean the previous data length for update
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1354) 			 * operation
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1355) 			 */
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1356) 			*prev_len = 0;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1357) 		} else {
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1358) 			areq_ctx->buff_index ^= 1;
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1359) 		}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1360) 	}
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1361) }
63893811b0fcb (Gilad Ben-Yossef   2018-01-22 09:27:02 +0000 1362) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1363) int cc_buffer_mgr_init(struct cc_drvdata *drvdata)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1364) {
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1365) 	struct device *dev = drvdata_to_dev(drvdata);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1366) 
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1367) 	drvdata->mlli_buffs_pool =
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1368) 		dma_pool_create("dx_single_mlli_tables", dev,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1369) 				MAX_NUM_OF_TOTAL_MLLI_ENTRIES *
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1370) 				LLI_ENTRY_BYTE_SIZE,
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1371) 				MLLI_TABLE_MIN_ALIGNMENT, 0);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1372) 
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1373) 	if (!drvdata->mlli_buffs_pool)
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1374) 		return -ENOMEM;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1375) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1376) 	return 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1377) }
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1378) 
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1379) int cc_buffer_mgr_fini(struct cc_drvdata *drvdata)
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1380) {
040187a0faa65 (Geert Uytterhoeven 2020-02-11 19:19:14 +0100 1381) 	dma_pool_destroy(drvdata->mlli_buffs_pool);
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1382) 	return 0;
4c3f97276e156 (Gilad Ben-Yossef   2018-01-22 09:27:00 +0000 1383) }