^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * CAAM/SEC 4.x transport/backend driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * JobR backend functionality
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright 2008-2012 Freescale Semiconductor, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright 2019 NXP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/of_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/of_address.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "compat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "ctrl.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "regs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "jr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "desc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "intern.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct jr_driver_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /* List of Physical JobR's with the Driver */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct list_head jr_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) spinlock_t jr_alloc_lock; /* jr_list lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) } ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static struct jr_driver_data driver_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static DEFINE_MUTEX(algs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static unsigned int active_devs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static void register_algs(struct caam_drv_private_jr *jrpriv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) mutex_lock(&algs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) if (++active_devs != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) goto algs_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) caam_algapi_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) caam_algapi_hash_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) caam_pkc_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) jrpriv->hwrng = !caam_rng_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) caam_qi_algapi_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) algs_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) mutex_unlock(&algs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static void unregister_algs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) mutex_lock(&algs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (--active_devs != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) goto algs_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) caam_qi_algapi_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) caam_pkc_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) caam_algapi_hash_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) caam_algapi_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) algs_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) mutex_unlock(&algs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static void caam_jr_crypto_engine_exit(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct device *jrdev = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* Free the resources of crypto-engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) crypto_engine_exit(jrpriv->engine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static int caam_reset_hw_jr(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) unsigned int timeout = 100000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * mask interrupts since we are going to poll
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * for reset completion status
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /* initiate flush (required prior to reset) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) JRINT_ERR_HALT_INPROGRESS) && --timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) JRINT_ERR_HALT_COMPLETE || timeout == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* initiate reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) timeout = 100000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (timeout == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* unmask interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * Shutdown JobR independent of platform property code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static int caam_jr_shutdown(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) ret = caam_reset_hw_jr(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) tasklet_kill(&jrp->irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static int caam_jr_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct device *jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct caam_drv_private_jr *jrpriv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) jrdev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) jrpriv = dev_get_drvdata(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (jrpriv->hwrng)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) caam_rng_exit(jrdev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * Return EBUSY if job ring already allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (atomic_read(&jrpriv->tfm_count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) dev_err(jrdev, "Device is busy\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* Unregister JR-based RNG & crypto algorithms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unregister_algs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* Remove the node from Physical JobR list maintained by driver */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) spin_lock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) list_del(&jrpriv->list_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) spin_unlock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /* Release ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) ret = caam_jr_shutdown(jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) dev_err(jrdev, "Failed to shut down job ring\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* Main per-ring interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct device *dev = st_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) u32 irqstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * Check the output ring for ready responses, kick
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * tasklet if jobs done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) irqstate = rd_reg32(&jrp->rregs->jrintstatus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!irqstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * If JobR error, we got more development work to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * Flag a bug now, but we really need to shut down and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * restart the queue (and fix code).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (irqstate & JRINT_JR_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* mask valid interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* Have valid interrupt at this point, just ACK and trigger */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) wr_reg32(&jrp->rregs->jrintstatus, irqstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) tasklet_schedule(&jrp->irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Deferred service handler, run as interrupt-fired tasklet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static void caam_jr_dequeue(unsigned long devarg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) int hw_idx, sw_idx, i, head, tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct device *dev = (struct device *)devarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) u32 *userdesc, userstatus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) void *userarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) u32 outring_used = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) while (outring_used ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) (outring_used = rd_reg32(&jrp->rregs->outring_used))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) head = READ_ONCE(jrp->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) sw_idx = tail = jrp->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) hw_idx = jrp->out_ring_read_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) sw_idx = (tail + i) & (JOBR_DEPTH - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (jr_outentry_desc(jrp->outring, hw_idx) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) break; /* found */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /* we should never fail to find a matching descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Unmap just-run descriptor so we can post-process */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) dma_unmap_single(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) caam_dma_to_cpu(jr_outentry_desc(jrp->outring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) hw_idx)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) jrp->entinfo[sw_idx].desc_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* mark completed, avoid matching on a recycled desc addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) jrp->entinfo[sw_idx].desc_addr_dma = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /* Stash callback params */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) usercall = jrp->entinfo[sw_idx].callbk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) userarg = jrp->entinfo[sw_idx].cbkarg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) userstatus = caam32_to_cpu(jr_outentry_jrstatus(jrp->outring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) hw_idx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * Make sure all information from the job has been obtained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * before telling CAAM that the job has been removed from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * output ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) /* set done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) wr_reg32(&jrp->rregs->outring_rmvd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) (JOBR_DEPTH - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * if this job completed out-of-order, do not increment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * the tail. Otherwise, increment tail by 1 plus the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * number of subsequent jobs already completed out-of-order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (sw_idx == tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) tail = (tail + 1) & (JOBR_DEPTH - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) jrp->entinfo[tail].desc_addr_dma == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) jrp->tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /* Finally, execute user's callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) usercall(dev, userdesc, userstatus, userarg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) outring_used--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* reenable / unmask IRQs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * returns : pointer to the newly allocated physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * JobR dev can be written to if successful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) struct device *caam_jr_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct device *dev = ERR_PTR(-ENODEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) int min_tfm_cnt = INT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) int tfm_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) spin_lock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (list_empty(&driver_data.jr_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) spin_unlock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return ERR_PTR(-ENODEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) tfm_cnt = atomic_read(&jrpriv->tfm_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (tfm_cnt < min_tfm_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) min_tfm_cnt = tfm_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) min_jrpriv = jrpriv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!min_tfm_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (min_jrpriv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) atomic_inc(&min_jrpriv->tfm_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) dev = min_jrpriv->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) spin_unlock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) EXPORT_SYMBOL(caam_jr_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * caam_jr_free() - Free the Job Ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * @rdev: points to the dev that identifies the Job ring to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * be released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) void caam_jr_free(struct device *rdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) atomic_dec(&jrpriv->tfm_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) EXPORT_SYMBOL(caam_jr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * caam_jr_enqueue() - Enqueue a job descriptor head. Returns -EINPROGRESS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * if OK, -ENOSPC if the queue is full, -EIO if it cannot map the caller's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * descriptor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * @dev: struct device of the job ring to be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * @desc: points to a job descriptor that execute our request. All
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * descriptors (and all referenced data) must be in a DMAable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * region, and all data references must be physical addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * accessible to CAAM (i.e. within a PAMU window granted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * to it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * @cbk: pointer to a callback function to be invoked upon completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * of this request. This has the form:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * callback(struct device *dev, u32 *desc, u32 stat, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * where:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * dev: contains the job ring device that processed this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * response.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * desc: descriptor that initiated the request, same as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * "desc" being argued to caam_jr_enqueue().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * status: untranslated status received from CAAM. See the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * reference manual for a detailed description of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * error meaning, or see the JRSTA definitions in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * register header file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * areq: optional pointer to an argument passed with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * original request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * @areq: optional pointer to a user argument for use at callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) int caam_jr_enqueue(struct device *dev, u32 *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) void (*cbk)(struct device *dev, u32 *desc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) u32 status, void *areq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) void *areq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct caam_jrentry_info *head_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) int head, tail, desc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) dma_addr_t desc_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (dma_mapping_error(dev, desc_dma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) spin_lock_bh(&jrp->inplock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) head = jrp->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) tail = READ_ONCE(jrp->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (!jrp->inpring_avail ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) spin_unlock_bh(&jrp->inplock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) head_entry = &jrp->entinfo[head];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) head_entry->desc_addr_virt = desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) head_entry->desc_size = desc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) head_entry->callbk = (void *)cbk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) head_entry->cbkarg = areq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) head_entry->desc_addr_dma = desc_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) jr_inpentry_set(jrp->inpring, head, cpu_to_caam_dma(desc_dma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Guarantee that the descriptor's DMA address has been written to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * the next slot in the ring before the write index is updated, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * other cores may update this index independently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) jrp->head = (head + 1) & (JOBR_DEPTH - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * Ensure that all job information has been written before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * notifying CAAM that a new job was added to the input ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * using a memory barrier. The wr_reg32() uses api iowrite32()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * to do the register write. iowrite32() issues a memory barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * before the write operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) wr_reg32(&jrp->rregs->inpring_jobadd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) jrp->inpring_avail--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (!jrp->inpring_avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) jrp->inpring_avail = rd_reg32(&jrp->rregs->inpring_avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) spin_unlock_bh(&jrp->inplock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) return -EINPROGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) EXPORT_SYMBOL(caam_jr_enqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * Init JobR independent of platform property detection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static int caam_jr_init(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct caam_drv_private_jr *jrp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) dma_addr_t inpbusaddr, outbusaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) int i, error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) jrp = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) error = caam_reset_hw_jr(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) jrp->inpring = dmam_alloc_coherent(dev, SIZEOF_JR_INPENTRY *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) JOBR_DEPTH, &inpbusaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (!jrp->inpring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) jrp->outring = dmam_alloc_coherent(dev, SIZEOF_JR_OUTENTRY *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) JOBR_DEPTH, &outbusaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (!jrp->outring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) jrp->entinfo = devm_kcalloc(dev, JOBR_DEPTH, sizeof(*jrp->entinfo),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (!jrp->entinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) for (i = 0; i < JOBR_DEPTH; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) jrp->entinfo[i].desc_addr_dma = !0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /* Setup rings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) jrp->out_ring_read_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) jrp->head = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) jrp->tail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) wr_reg64(&jrp->rregs->outring_base, outbusaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) jrp->inpring_avail = JOBR_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) spin_lock_init(&jrp->inplock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) /* Select interrupt coalescing parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* Connect job ring interrupt handler. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) dev_name(dev), dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) jrp->ridx, jrp->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) tasklet_kill(&jrp->irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static void caam_jr_irq_dispose_mapping(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) irq_dispose_mapping((unsigned long)data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * Probe routine for each detected JobR subsystem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static int caam_jr_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct device *jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct device_node *nprop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) struct caam_job_ring __iomem *ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) struct caam_drv_private_jr *jrpriv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static int total_jobrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct resource *r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) jrdev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) jrpriv = devm_kzalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (!jrpriv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) dev_set_drvdata(jrdev, jrpriv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /* save ring identity relative to detection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) jrpriv->ridx = total_jobrs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) nprop = pdev->dev.of_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /* Get configuration properties from device tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* First, get register page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (!r) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) dev_err(jrdev, "platform_get_resource() failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) ctrl = devm_ioremap(jrdev, r->start, resource_size(r));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (!ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) dev_err(jrdev, "devm_ioremap() failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) error = dma_set_mask_and_coherent(jrdev, caam_get_dma_mask(jrdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /* Initialize crypto engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) jrpriv->engine = crypto_engine_alloc_init(jrdev, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (!jrpriv->engine) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) dev_err(jrdev, "Could not init crypto-engine\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) error = devm_add_action_or_reset(jrdev, caam_jr_crypto_engine_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) jrdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) /* Start crypto engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) error = crypto_engine_start(jrpriv->engine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) dev_err(jrdev, "Could not start crypto-engine\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /* Identify the interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) jrpriv->irq = irq_of_parse_and_map(nprop, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (!jrpriv->irq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) dev_err(jrdev, "irq_of_parse_and_map failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) error = devm_add_action_or_reset(jrdev, caam_jr_irq_dispose_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) (void *)(unsigned long)jrpriv->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /* Now do the platform independent part */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) error = caam_jr_init(jrdev); /* now turn on hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) jrpriv->dev = jrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) spin_lock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) spin_unlock(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) atomic_set(&jrpriv->tfm_count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) register_algs(jrpriv, jrdev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) static const struct of_device_id caam_jr_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) .compatible = "fsl,sec-v4.0-job-ring",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) .compatible = "fsl,sec4.0-job-ring",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) MODULE_DEVICE_TABLE(of, caam_jr_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) static struct platform_driver caam_jr_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) .name = "caam_jr",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) .of_match_table = caam_jr_match,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) .probe = caam_jr_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) .remove = caam_jr_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static int __init jr_driver_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) spin_lock_init(&driver_data.jr_alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) INIT_LIST_HEAD(&driver_data.jr_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return platform_driver_register(&caam_jr_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) static void __exit jr_driver_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) platform_driver_unregister(&caam_jr_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) module_init(jr_driver_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) module_exit(jr_driver_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) MODULE_DESCRIPTION("FSL CAAM JR request backend");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags