^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * AMD Cryptographic Coprocessor (CCP) driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Tom Lendacky <thomas.lendacky@amd.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Gary R Hook <gary.hook@amd.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/ccp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "ccp-dev.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static u32 ccp_alloc_ksb(struct ccp_cmd_queue *cmd_q, unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct ccp_device *ccp = cmd_q->ccp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) mutex_lock(&ccp->sb_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) start = (u32)bitmap_find_next_zero_area(ccp->sb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) ccp->sb_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) ccp->sb_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) if (start <= ccp->sb_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) bitmap_set(ccp->sb, start, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) mutex_unlock(&ccp->sb_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) ccp->sb_avail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) mutex_unlock(&ccp->sb_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Wait for KSB entries to become available */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return KSB_START + start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static void ccp_free_ksb(struct ccp_cmd_queue *cmd_q, unsigned int start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct ccp_device *ccp = cmd_q->ccp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (!start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) mutex_lock(&ccp->sb_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) bitmap_clear(ccp->sb, start - KSB_START, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) ccp->sb_avail = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) mutex_unlock(&ccp->sb_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) wake_up_interruptible_all(&ccp->sb_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static unsigned int ccp_get_free_slots(struct ccp_cmd_queue *cmd_q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) return CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct ccp_cmd_queue *cmd_q = op->cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct ccp_device *ccp = cmd_q->ccp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) void __iomem *cr_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) u32 cr0, cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* We could read a status register to see how many free slots
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * are actually available, but reading that register resets it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * and you could lose some error information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) cmd_q->free_slots--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) | (op->jobid << REQ0_JOBID_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) | REQ0_WAIT_FOR_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (op->soc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) cr0 |= REQ0_STOP_ON_COMPLETE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) | REQ0_INT_ON_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if (op->ioc || !cmd_q->free_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) cr0 |= REQ0_INT_ON_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* Start at CMD_REQ1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) mutex_lock(&ccp->req_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* Write CMD_REQ1 through CMD_REQx first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) iowrite32(*(cr + i), cr_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /* Tell the CCP to start */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) iowrite32(cr0, ccp->io_regs + CMD_REQ0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) mutex_unlock(&ccp->req_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (cr0 & REQ0_INT_ON_COMPLETE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* Wait for the job to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ret = wait_event_interruptible(cmd_q->int_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) cmd_q->int_rcvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (ret || cmd_q->cmd_error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* On error delete all related jobs from the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) | op->jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (cmd_q->cmd_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) ccp_log_error(cmd_q->ccp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) cmd_q->cmd_error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) } else if (op->soc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) /* Delete just head job from the queue on SoC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) cmd = DEL_Q_ACTIVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) | (cmd_q->id << DEL_Q_ID_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) | op->jobid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) cmd_q->int_rcvd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static int ccp_perform_aes(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) | (op->sb_key << REQ1_KEY_KSB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) cr[1] = op->src.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) cr[4] = ccp_addr_lo(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) | ccp_addr_hi(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (op->u.aes.mode == CCP_AES_MODE_CFB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (op->eom)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) cr[0] |= REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (op->init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) cr[0] |= REQ1_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static int ccp_perform_xts_aes(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) | (op->sb_key << REQ1_KEY_KSB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) cr[1] = op->src.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) cr[4] = ccp_addr_lo(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) | ccp_addr_hi(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (op->eom)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) cr[0] |= REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (op->init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) cr[0] |= REQ1_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static int ccp_perform_sha(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) | REQ1_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) cr[1] = op->src.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (op->eom) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) cr[0] |= REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) cr[4] = lower_32_bits(op->u.sha.msg_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) cr[5] = upper_32_bits(op->u.sha.msg_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) cr[4] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) cr[5] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static int ccp_perform_rsa(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) | (op->sb_key << REQ1_KEY_KSB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) | REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) cr[1] = op->u.rsa.input_len - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) cr[3] = (op->sb_ctx << REQ4_KSB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) cr[4] = ccp_addr_lo(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) | ccp_addr_hi(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static int ccp_perform_passthru(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (op->src.type == CCP_MEMTYPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) cr[1] = op->src.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) cr[1] = op->dst.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (op->src.type == CCP_MEMTYPE_SYSTEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) cr[3] |= (op->sb_key << REQ4_KSB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) cr[2] = op->src.u.sb * CCP_SB_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) cr[3] = (CCP_MEMTYPE_SB << REQ4_MEMTYPE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) cr[4] = ccp_addr_lo(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) | ccp_addr_hi(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) cr[4] = op->dst.u.sb * CCP_SB_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) cr[5] = (CCP_MEMTYPE_SB << REQ6_MEMTYPE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (op->eom)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) cr[0] |= REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static int ccp_perform_ecc(struct ccp_op *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) u32 cr[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /* Fill out the register contents for REQ1 through REQ6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) cr[0] = REQ1_ECC_AFFINE_CONVERT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) | REQ1_EOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) cr[1] = op->src.u.dma.length - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) cr[2] = ccp_addr_lo(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) | ccp_addr_hi(&op->src.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) cr[4] = ccp_addr_lo(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) | ccp_addr_hi(&op->dst.u.dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static void ccp_disable_queue_interrupts(struct ccp_device *ccp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void ccp_enable_queue_interrupts(struct ccp_device *ccp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) iowrite32(ccp->qim, ccp->io_regs + IRQ_MASK_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static void ccp_irq_bh(unsigned long data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct ccp_device *ccp = (struct ccp_device *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct ccp_cmd_queue *cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) u32 q_int, status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) status = ioread32(ccp->io_regs + IRQ_STATUS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) for (i = 0; i < ccp->cmd_q_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) cmd_q = &ccp->cmd_q[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) q_int = status & (cmd_q->int_ok | cmd_q->int_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (q_int) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) cmd_q->int_status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) cmd_q->q_status = ioread32(cmd_q->reg_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) /* On error, only save the first error value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) cmd_q->int_rcvd = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /* Acknowledge the interrupt and wake the kthread */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) wake_up_interruptible(&cmd_q->int_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) ccp_enable_queue_interrupts(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static irqreturn_t ccp_irq_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct ccp_device *ccp = (struct ccp_device *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ccp_disable_queue_interrupts(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (ccp->use_tasklet)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) tasklet_schedule(&ccp->irq_tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) ccp_irq_bh((unsigned long)ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) static int ccp_init(struct ccp_device *ccp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct device *dev = ccp->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct ccp_cmd_queue *cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct dma_pool *dma_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) unsigned int qmr, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* Find available queues */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ccp->qim = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) qmr = ioread32(ccp->io_regs + Q_MASK_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) for (i = 0; (i < MAX_HW_QUEUES) && (ccp->cmd_q_count < ccp->max_q_count); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!(qmr & (1 << i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* Allocate a dma pool for this queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ccp->name, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) dma_pool = dma_pool_create(dma_pool_name, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) CCP_DMAPOOL_MAX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) CCP_DMAPOOL_ALIGN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (!dma_pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) dev_err(dev, "unable to allocate dma pool\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) goto e_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) ccp->cmd_q_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) cmd_q->ccp = ccp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) cmd_q->id = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) cmd_q->dma_pool = dma_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* Reserve 2 KSB regions for the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) cmd_q->sb_key = KSB_START + ccp->sb_start++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) cmd_q->sb_ctx = KSB_START + ccp->sb_start++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) ccp->sb_count -= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /* Preset some register values and masks that are queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * number dependent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) (CMD_Q_STATUS_INCR * i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) (CMD_Q_STATUS_INCR * i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) cmd_q->int_ok = 1 << (i * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) cmd_q->int_err = 1 << ((i * 2) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) cmd_q->free_slots = ccp_get_free_slots(cmd_q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) init_waitqueue_head(&cmd_q->int_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) /* Build queue interrupt mask (two interrupts per queue) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) ccp->qim |= cmd_q->int_ok | cmd_q->int_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) #ifdef CONFIG_ARM64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /* For arm64 set the recommended queue cache settings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) (CMD_Q_CACHE_INC * i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) dev_dbg(dev, "queue #%u available\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (ccp->cmd_q_count == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) dev_notice(dev, "no command queues available\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) goto e_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /* Disable and clear interrupts until ready */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) ccp_disable_queue_interrupts(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) for (i = 0; i < ccp->cmd_q_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) cmd_q = &ccp->cmd_q[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) ioread32(cmd_q->reg_int_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) ioread32(cmd_q->reg_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /* Request an irq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) dev_err(dev, "unable to allocate an IRQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) goto e_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* Initialize the ISR tasklet? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (ccp->use_tasklet)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) tasklet_init(&ccp->irq_tasklet, ccp_irq_bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) (unsigned long)ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) dev_dbg(dev, "Starting threads...\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* Create a kthread for each queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) for (i = 0; i < ccp->cmd_q_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct task_struct *kthread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) cmd_q = &ccp->cmd_q[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) "%s-q%u", ccp->name, cmd_q->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (IS_ERR(kthread)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) dev_err(dev, "error creating queue thread (%ld)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) PTR_ERR(kthread));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) ret = PTR_ERR(kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) goto e_kthread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) cmd_q->kthread = kthread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) wake_up_process(kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) dev_dbg(dev, "Enabling interrupts...\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /* Enable interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) ccp_enable_queue_interrupts(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) dev_dbg(dev, "Registering device...\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) ccp_add_device(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ret = ccp_register_rng(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) goto e_kthread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /* Register the DMA engine support */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) ret = ccp_dmaengine_register(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) goto e_hwrng;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) e_hwrng:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) ccp_unregister_rng(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) e_kthread:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) for (i = 0; i < ccp->cmd_q_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (ccp->cmd_q[i].kthread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) kthread_stop(ccp->cmd_q[i].kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) sp_free_ccp_irq(ccp->sp, ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) e_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) for (i = 0; i < ccp->cmd_q_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) dma_pool_destroy(ccp->cmd_q[i].dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) static void ccp_destroy(struct ccp_device *ccp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) struct ccp_cmd_queue *cmd_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct ccp_cmd *cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* Unregister the DMA engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) ccp_dmaengine_unregister(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* Unregister the RNG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) ccp_unregister_rng(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* Remove this device from the list of available units */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) ccp_del_device(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /* Disable and clear interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) ccp_disable_queue_interrupts(ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) for (i = 0; i < ccp->cmd_q_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) cmd_q = &ccp->cmd_q[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ioread32(cmd_q->reg_int_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ioread32(cmd_q->reg_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /* Stop the queue kthreads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) for (i = 0; i < ccp->cmd_q_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (ccp->cmd_q[i].kthread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) kthread_stop(ccp->cmd_q[i].kthread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) sp_free_ccp_irq(ccp->sp, ccp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) for (i = 0; i < ccp->cmd_q_count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) dma_pool_destroy(ccp->cmd_q[i].dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Flush the cmd and backlog queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) while (!list_empty(&ccp->cmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) /* Invoke the callback directly with an error code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) list_del(&cmd->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) cmd->callback(cmd->data, -ENODEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) while (!list_empty(&ccp->backlog)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* Invoke the callback directly with an error code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) list_del(&cmd->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) cmd->callback(cmd->data, -ENODEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) static const struct ccp_actions ccp3_actions = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) .aes = ccp_perform_aes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) .xts_aes = ccp_perform_xts_aes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) .des3 = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) .sha = ccp_perform_sha,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) .rsa = ccp_perform_rsa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) .passthru = ccp_perform_passthru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) .ecc = ccp_perform_ecc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) .sballoc = ccp_alloc_ksb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) .sbfree = ccp_free_ksb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) .init = ccp_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) .destroy = ccp_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .get_free_slots = ccp_get_free_slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .irqhandler = ccp_irq_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) const struct ccp_vdata ccpv3_platform = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) .version = CCP_VERSION(3, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) .setup = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) .perform = &ccp3_actions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) .offset = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .rsamax = CCP_RSA_MAX_WIDTH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) const struct ccp_vdata ccpv3 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) .version = CCP_VERSION(3, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) .setup = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) .perform = &ccp3_actions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) .offset = 0x20000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) .rsamax = CCP_RSA_MAX_WIDTH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate