Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Provide TDMA helper functions used by cipher and hash algorithm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * implementations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Author: Arnaud Ebalard <arno@natisbad.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * This work is based on an initial version written by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include "cesa.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 					struct mv_cesa_sg_dma_iter *sgiter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 					unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	if (!sgiter->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	sgiter->op_offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	sgiter->offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	if (sgiter->offset == sg_dma_len(sgiter->sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 		if (sg_is_last(sgiter->sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 		sgiter->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		sgiter->sg = sg_next(sgiter->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	if (sgiter->op_offset == iter->op_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) void mv_cesa_dma_step(struct mv_cesa_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	struct mv_cesa_engine *engine = dreq->engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	writel_relaxed(0, engine->regs + CESA_SA_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 		       CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 		       engine->regs + CESA_TDMA_CONTROL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 		       CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 		       engine->regs + CESA_SA_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	writel_relaxed(dreq->chain.first->cur_dma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		       engine->regs + CESA_TDMA_NEXT_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	WARN_ON(readl(engine->regs + CESA_SA_CMD) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		CESA_SA_CMD_EN_CESA_SA_ACCL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) void mv_cesa_dma_cleanup(struct mv_cesa_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	for (tdma = dreq->chain.first; tdma;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 		struct mv_cesa_tdma_desc *old_tdma = tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 		u32 type = tdma->flags & CESA_TDMA_TYPE_MSK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		if (type == CESA_TDMA_OP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 			dma_pool_free(cesa_dev->dma->op_pool, tdma->op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 				      le32_to_cpu(tdma->src));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		tdma = tdma->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 			      old_tdma->cur_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	dreq->chain.first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	dreq->chain.last = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) void mv_cesa_dma_prepare(struct mv_cesa_req *dreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 			 struct mv_cesa_engine *engine)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	for (tdma = dreq->chain.first; tdma; tdma = tdma->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		if (tdma->flags & CESA_TDMA_DST_IN_SRAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 			tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 		if (tdma->flags & CESA_TDMA_SRC_IN_SRAM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 			tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 			mv_cesa_adjust_op(engine, tdma->op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) void mv_cesa_tdma_chain(struct mv_cesa_engine *engine,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 			struct mv_cesa_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	if (engine->chain.first == NULL && engine->chain.last == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		engine->chain.first = dreq->chain.first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		engine->chain.last  = dreq->chain.last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		struct mv_cesa_tdma_desc *last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		last = engine->chain.last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		last->next = dreq->chain.first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		engine->chain.last = dreq->chain.last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 		 * Break the DMA chain if the CESA_TDMA_BREAK_CHAIN is set on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 		 * the last element of the current chain, or if the request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		 * being queued needs the IV regs to be set before lauching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		 * the request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 		if (!(last->flags & CESA_TDMA_BREAK_CHAIN) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		    !(dreq->chain.first->flags & CESA_TDMA_SET_STATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 			last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct crypto_async_request *req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	struct mv_cesa_tdma_desc *tdma = NULL, *next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	dma_addr_t tdma_cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	int res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	tdma_cur = readl(engine->regs + CESA_TDMA_CUR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	for (tdma = engine->chain.first; tdma; tdma = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		spin_lock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		next = tdma->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		spin_unlock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		if (tdma->flags & CESA_TDMA_END_OF_REQ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 			struct crypto_async_request *backlog = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 			struct mv_cesa_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 			u32 current_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 			spin_lock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 			 * if req is NULL, this means we're processing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 			 * request in engine->req.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 			if (!req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 				req = engine->req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 			else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 				req = mv_cesa_dequeue_req_locked(engine,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 								 &backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 			/* Re-chaining to the next request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 			engine->chain.first = tdma->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 			tdma->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 			/* If this is the last request, clear the chain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 			if (engine->chain.first == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 				engine->chain.last  = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 			spin_unlock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			ctx = crypto_tfm_ctx(req->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 			current_status = (tdma->cur_dma == tdma_cur) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 					  status : CESA_SA_INT_ACC0_IDMA_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 			res = ctx->ops->process(req, current_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 			ctx->ops->complete(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 			if (res == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 				mv_cesa_engine_enqueue_complete_request(engine,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 									req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 			if (backlog)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 				backlog->complete(backlog, -EINPROGRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		if (res || tdma->cur_dma == tdma_cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	 * Save the last request in error to engine->req, so that the core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 * knows which request was fautly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	if (res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		spin_lock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		engine->req = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		spin_unlock_bh(&engine->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static struct mv_cesa_tdma_desc *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	struct mv_cesa_tdma_desc *new_tdma = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	dma_addr_t dma_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 				   &dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	if (!new_tdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 		return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	new_tdma->cur_dma = dma_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	if (chain->last) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		chain->last->next_dma = cpu_to_le32(dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 		chain->last->next = new_tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 		chain->first = new_tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	chain->last = new_tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	return new_tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) int mv_cesa_dma_add_result_op(struct mv_cesa_tdma_chain *chain, dma_addr_t src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 			  u32 size, u32 flags, gfp_t gfp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	struct mv_cesa_tdma_desc *tdma, *op_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	tdma = mv_cesa_dma_add_desc(chain, gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	if (IS_ERR(tdma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		return PTR_ERR(tdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	/* We re-use an existing op_desc object to retrieve the context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	 * and result instead of allocating a new one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	 * There is at least one object of this type in a CESA crypto
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	 * req, just pick the first one in the chain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	for (op_desc = chain->first; op_desc; op_desc = op_desc->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 		if (type == CESA_TDMA_OP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	if (!op_desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	tdma->src_dma = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	tdma->dst_dma = op_desc->src_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	tdma->op = op_desc->op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	tdma->flags = flags | CESA_TDMA_RESULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 					const struct mv_cesa_op_ctx *op_templ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 					bool skip_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 					gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	struct mv_cesa_op_ctx *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	dma_addr_t dma_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	tdma = mv_cesa_dma_add_desc(chain, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	if (IS_ERR(tdma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		return ERR_CAST(tdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	if (!op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	*op = *op_templ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	size = skip_ctx ? sizeof(op->desc) : sizeof(*op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	tdma = chain->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	tdma->op = op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	tdma->src = cpu_to_le32(dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	return op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 				  dma_addr_t dst, dma_addr_t src, u32 size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 				  u32 flags, gfp_t gfp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	tdma = mv_cesa_dma_add_desc(chain, gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	if (IS_ERR(tdma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		return PTR_ERR(tdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	tdma->src_dma = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	tdma->dst_dma = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	tdma->flags = flags | CESA_TDMA_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	tdma = mv_cesa_dma_add_desc(chain, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	return PTR_ERR_OR_ZERO(tdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	struct mv_cesa_tdma_desc *tdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	tdma = mv_cesa_dma_add_desc(chain, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	if (IS_ERR(tdma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		return PTR_ERR(tdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	tdma->byte_cnt = cpu_to_le32(BIT(31));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 				 struct mv_cesa_dma_iter *dma_iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 				 struct mv_cesa_sg_dma_iter *sgiter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 				 gfp_t gfp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	u32 flags = sgiter->dir == DMA_TO_DEVICE ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 		    CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 		dma_addr_t dst, src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		if (sgiter->dir == DMA_TO_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 			dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 			src = sg_dma_address(sgiter->sg) + sgiter->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 			dst = sg_dma_address(sgiter->sg) + sgiter->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 			src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 		ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 						    flags, gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 			return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	} while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }