^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * CAAM/SEC 4.x QI transport/backend driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Queue Interface backend functionality
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright 2013-2016 Freescale Semiconductor, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright 2016-2017, 2019-2020 NXP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <soc/fsl/qman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "debugfs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "regs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "qi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "desc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "intern.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "desc_constr.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define PREHDR_RSLS_SHIFT 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define PREHDR_ABS BIT(25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Use a reasonable backlog of frames (per CPU) as congestion threshold,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * so that resources used by the in-flight buffers do not become a memory hog.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define MAX_RSP_FQ_BACKLOG_PER_CPU 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define CAAM_QI_ENQUEUE_RETRIES 10000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define CAAM_NAPI_WEIGHT 63
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * caam_napi - struct holding CAAM NAPI-related params
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * @irqtask: IRQ task for QI backend
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * @p: QMan portal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct caam_napi {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct napi_struct irqtask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * caam_qi_pcpu_priv - percpu private data structure to main list of pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * responses expected on each cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * @caam_napi: CAAM NAPI params
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * @net_dev: netdev used by NAPI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * @rsp_fq: response FQ from CAAM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct caam_qi_pcpu_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct caam_napi caam_napi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct net_device net_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct qman_fq *rsp_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) } ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static DEFINE_PER_CPU(int, last_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * caam_qi_priv - CAAM QI backend private params
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * @cgr: QMan congestion group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct caam_qi_priv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct qman_cgr cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static struct caam_qi_priv qipriv ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * This is written by only one core - the one that initialized the CGR - and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * read by multiple cores (all the others).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) bool caam_congested __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) EXPORT_SYMBOL(caam_congested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * This is a a cache of buffers, from which the users of CAAM QI driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * doing malloc on the hotpath.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * NOTE: A more elegant solution would be to have some headroom in the frames
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * being processed. This could be added by the dpaa-ethernet driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * This would pose a problem for userspace application processing which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * cannot know of this limitation. So for now, this will work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static struct kmem_cache *qi_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static void *caam_iova_to_virt(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) dma_addr_t iova_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) phys_addr_t phys_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return phys_to_virt(phys_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct qm_fd fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) dma_addr_t addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int num_retries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) qm_fd_clear_fd(&fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) qm_fd_set_compound(&fd, qm_sg_entry_get_len(&req->fd_sgt[1]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (dma_mapping_error(qidev, addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) dev_err(qidev, "DMA mapping error for QI enqueue request\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) qm_fd_addr_set64(&fd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ret = qman_enqueue(req->drv_ctx->req_fq, &fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (likely(!ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) refcount_inc(&req->drv_ctx->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (ret != -EBUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) num_retries++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) } while (num_retries < CAAM_QI_ENQUEUE_RETRIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) dev_err(qidev, "qman_enqueue failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) EXPORT_SYMBOL(caam_qi_enqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) const union qm_mr_entry *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) const struct qm_fd *fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct caam_drv_req *drv_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct caam_drv_private *priv = dev_get_drvdata(qidev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) fd = &msg->ern.fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (!drv_req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) dev_err(qidev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) "Can't find original request for CAAM response\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) refcount_dec(&drv_req->drv_ctx->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (qm_fd_get_format(fd) != qm_fd_compound) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) dev_err(qidev, "Non-compound FD from CAAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (fd->status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) drv_req->cbk(drv_req, be32_to_cpu(fd->status));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) drv_req->cbk(drv_req, JRSTA_SSRC_QI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static struct qman_fq *create_caam_req_fq(struct device *qidev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct qman_fq *rsp_fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) dma_addr_t hwdesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int fq_sched_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct qman_fq *req_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct qm_mcc_initfq opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (!req_fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) req_fq->cb.ern = caam_fq_ern_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) req_fq->cb.fqs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) QMAN_FQ_FLAG_TO_DCPORTAL, req_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) dev_err(qidev, "Failed to create session req FQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) goto create_req_fq_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) memset(&opts, 0, sizeof(opts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) QM_INITFQ_WE_CONTEXTB |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) qm_fqd_set_destwq(&opts.fqd, qm_channel_caam, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) opts.fqd.context_b = cpu_to_be32(qman_fq_fqid(rsp_fq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) qm_fqd_context_a_set64(&opts.fqd, hwdesc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) opts.fqd.cgid = qipriv.cgr.cgrid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) ret = qman_init_fq(req_fq, fq_sched_flag, &opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) dev_err(qidev, "Failed to init session req FQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) goto init_req_fq_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return req_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) init_req_fq_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) qman_destroy_fq(req_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) create_req_fq_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) kfree(req_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static int empty_retired_fq(struct device *qidev, struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) QMAN_VOLATILE_FLAG_FINISH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) QM_VDQCR_PRECEDENCE_VDQCR |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) QM_VDQCR_NUMFRAMES_TILLEMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) p = qman_get_affine_portal(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) qman_p_poll_dqrr(p, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) } while (fq->flags & QMAN_FQ_STATE_NE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static int kill_fq(struct device *qidev, struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ret = qman_retire_fq(fq, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) dev_err(qidev, "qman_retire_fq failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) goto empty_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) /* Async FQ retirement condition */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (ret == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /* Retry till FQ gets in retired state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) msleep(20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) } while (fq->state != qman_fq_state_retired);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) WARN_ON(fq->flags & QMAN_FQ_STATE_ORL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) empty_fq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (fq->flags & QMAN_FQ_STATE_NE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) ret = empty_retired_fq(qidev, fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) dev_err(qidev, "empty_retired_fq fail for FQ: %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) ret = qman_oos_fq(fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) qman_destroy_fq(fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) kfree(fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) static int empty_caam_fq(struct qman_fq *fq, struct caam_drv_ctx *drv_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int retries = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct qm_mcr_queryfq_np np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /* Wait till the older CAAM FQ get empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) ret = qman_query_fq_np(fq, &np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!qm_mcr_np_get(&np, frm_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) msleep(20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /* Wait until pending jobs from this FQ are processed by CAAM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (refcount_read(&drv_ctx->refcnt) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) msleep(20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) } while (--retries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (!retries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) dev_warn_once(drv_ctx->qidev, "%d frames from FQID %u still pending in CAAM\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) refcount_read(&drv_ctx->refcnt), fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) u32 num_words;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct qman_fq *new_fq, *old_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) struct device *qidev = drv_ctx->qidev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) num_words = desc_len(sh_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (num_words > MAX_SDLEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) dev_err(qidev, "Invalid descriptor len: %d words\n", num_words);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* Note down older req FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) old_fq = drv_ctx->req_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /* Create a new req FQ in parked state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) drv_ctx->context_a, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (IS_ERR(new_fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) dev_err(qidev, "FQ allocation for shdesc update failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return PTR_ERR(new_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) /* Hook up new FQ to context so that new requests keep queuing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) drv_ctx->req_fq = new_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /* Empty and remove the older FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ret = empty_caam_fq(old_fq, drv_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) /* We can revert to older FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) drv_ctx->req_fq = old_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (kill_fq(qidev, new_fq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) dev_warn(qidev, "New CAAM FQ kill failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * Re-initialise pre-header. Set RSLS and SDLEN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * Update the shared descriptor for driver context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) num_words);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) dma_sync_single_for_device(qidev, drv_ctx->context_a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) sizeof(drv_ctx->sh_desc) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) sizeof(drv_ctx->prehdr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /* Put the new FQ in scheduled state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ret = qman_schedule_fq(new_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * We can kill new FQ and revert to old FQ.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * Since the desc is already modified, it is success case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) drv_ctx->req_fq = old_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (kill_fq(qidev, new_fq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) dev_warn(qidev, "New CAAM FQ kill failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) } else if (kill_fq(qidev, old_fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) dev_warn(qidev, "Old CAAM FQ kill failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) EXPORT_SYMBOL(caam_drv_ctx_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) int *cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) u32 *sh_desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) u32 num_words;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) dma_addr_t hwdesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) struct caam_drv_ctx *drv_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) const cpumask_t *cpus = qman_affine_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) num_words = desc_len(sh_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (num_words > MAX_SDLEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) dev_err(qidev, "Invalid descriptor len: %d words\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) num_words);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (!drv_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * Initialise pre-header - set RSLS and SDLEN - and shared descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * and dma-map them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) num_words);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (dma_mapping_error(qidev, hwdesc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) dev_err(qidev, "DMA map error for preheader + shdesc\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) kfree(drv_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) drv_ctx->context_a = hwdesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /* If given CPU does not own the portal, choose another one that does */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (!cpumask_test_cpu(*cpu, cpus)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) int *pcpu = &get_cpu_var(last_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) *pcpu = cpumask_next(*pcpu, cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (*pcpu >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) *pcpu = cpumask_first(cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) *cpu = *pcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) put_cpu_var(last_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) drv_ctx->cpu = *cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /* Find response FQ hooked with this CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Attach request FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) QMAN_INITFQ_FLAG_SCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (IS_ERR(drv_ctx->req_fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) dev_err(qidev, "create_caam_req_fq failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) kfree(drv_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /* init reference counter used to track references to request FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) refcount_set(&drv_ctx->refcnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) drv_ctx->qidev = qidev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) return drv_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) EXPORT_SYMBOL(caam_drv_ctx_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) void *qi_cache_alloc(gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return kmem_cache_alloc(qi_cache, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) EXPORT_SYMBOL(qi_cache_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) void qi_cache_free(void *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) kmem_cache_free(qi_cache, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) EXPORT_SYMBOL(qi_cache_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) static int caam_qi_poll(struct napi_struct *napi, int budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct caam_napi *np = container_of(napi, struct caam_napi, irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) int cleaned = qman_p_poll_dqrr(np->p, budget);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (cleaned < budget) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) napi_complete(napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return cleaned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (IS_ERR_OR_NULL(drv_ctx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /* Remove request FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) kfree(drv_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) EXPORT_SYMBOL(caam_drv_ctx_rel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static void caam_qi_shutdown(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) struct device *qidev = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) struct caam_qi_priv *priv = &qipriv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) const cpumask_t *cpus = qman_affine_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) for_each_cpu(i, cpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct napi_struct *irqtask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) napi_disable(irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) netif_napi_del(irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) qman_delete_cgr_safe(&priv->cgr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) qman_release_cgrid(priv->cgr.cgrid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) kmem_cache_destroy(qi_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) caam_congested = congested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (congested) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) caam_debugfs_qi_congested();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) pr_debug_ratelimited("CAAM entered congestion\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) pr_debug_ratelimited("CAAM exited congestion\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * In case of threaded ISR, for RT kernels in_irq() does not return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * appropriate value, so use in_serving_softirq to distinguish between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * softirq and irq contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (unlikely(in_irq() || !in_serving_softirq())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* Disable QMan IRQ source and invoke NAPI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) qman_p_irqsource_remove(p, QM_PIRQ_DQRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) np->p = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) napi_schedule(&np->irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct qman_fq *rsp_fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) const struct qm_dqrr_entry *dqrr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) struct caam_drv_req *drv_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) const struct qm_fd *fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct caam_drv_private *priv = dev_get_drvdata(qidev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (caam_qi_napi_schedule(p, caam_napi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return qman_cb_dqrr_stop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) fd = &dqrr->fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (unlikely(!drv_req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) dev_err(qidev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) "Can't find original request for caam response\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return qman_cb_dqrr_consume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) refcount_dec(&drv_req->drv_ctx->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) status = be32_to_cpu(fd->status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (unlikely(status)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) u32 ssrc = status & JRSTA_SSRC_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (ssrc != JRSTA_SSRC_CCB_ERROR ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) err_id != JRSTA_CCBERR_ERRID_ICVCHK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) dev_err_ratelimited(qidev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) "Error: %#x in CAAM response FD\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (unlikely(qm_fd_get_format(fd) != qm_fd_compound)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) dev_err(qidev, "Non-compound FD from CAAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return qman_cb_dqrr_consume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) drv_req->cbk(drv_req, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return qman_cb_dqrr_consume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct qm_mcc_initfq opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct qman_fq *fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (!fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) fq->cb.dqrr = caam_rsp_fq_dqrr_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) QMAN_FQ_FLAG_DYNAMIC_FQID, fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) dev_err(qidev, "Rsp FQ create failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) kfree(fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) memset(&opts, 0, sizeof(opts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) opts.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) QM_INITFQ_WE_CONTEXTB |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CGID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) opts.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_CTXASTASHING |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) qm_fqd_set_destwq(&opts.fqd, qman_affine_channel(cpu), 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) opts.fqd.cgid = qipriv.cgr.cgrid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) QM_STASHING_EXCL_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) qm_fqd_set_stashing(&opts.fqd, 0, 1, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) dev_err(qidev, "Rsp FQ init failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) kfree(fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) static int init_cgr(struct device *qidev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) struct qm_mcc_initcgr opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) const u64 val = (u64)cpumask_weight(qman_affine_cpus()) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) MAX_RSP_FQ_BACKLOG_PER_CPU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) ret = qman_alloc_cgrid(&qipriv.cgr.cgrid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) qipriv.cgr.cb = cgr_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) memset(&opts, 0, sizeof(opts));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) QM_CGR_WE_MODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) opts.cgr.cscn_en = QM_CGR_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) opts.cgr.mode = QMAN_CGR_MODE_FRAME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) qipriv.cgr.cgrid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) dev_dbg(qidev, "Congestion threshold set to %llu\n", val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static int alloc_rsp_fqs(struct device *qidev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) const cpumask_t *cpus = qman_affine_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /*Now create response FQs*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) for_each_cpu(i, cpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) ret = alloc_rsp_fq_cpu(qidev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static void free_rsp_fqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) const cpumask_t *cpus = qman_affine_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) for_each_cpu(i, cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) kfree(per_cpu(pcpu_qipriv.rsp_fq, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) int caam_qi_init(struct platform_device *caam_pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) int err, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct device *ctrldev = &caam_pdev->dev, *qidev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct caam_drv_private *ctrlpriv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) const cpumask_t *cpus = qman_affine_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) ctrlpriv = dev_get_drvdata(ctrldev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) qidev = ctrldev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) /* Initialize the congestion detection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) err = init_cgr(qidev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) dev_err(qidev, "CGR initialization failed: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* Initialise response FQs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) err = alloc_rsp_fqs(qidev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) free_rsp_fqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * Enable the NAPI contexts on each of the core which has an affine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * portal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) for_each_cpu(i, cpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) struct caam_napi *caam_napi = &priv->caam_napi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) struct napi_struct *irqtask = &caam_napi->irqtask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) struct net_device *net_dev = &priv->net_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) net_dev->dev = *qidev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) INIT_LIST_HEAD(&net_dev->napi_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) netif_napi_add(net_dev, irqtask, caam_qi_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) CAAM_NAPI_WEIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) napi_enable(irqtask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) SLAB_CACHE_DMA, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (!qi_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) dev_err(qidev, "Can't allocate CAAM cache\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) free_rsp_fqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) caam_debugfs_qi_init(ctrlpriv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags