^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) // Copyright (c) 2018-2019 MediaTek Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Driver for MediaTek Command-Queue DMA Controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/iopoll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/of_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/of_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include "../virt-dma.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define MTK_CQDMA_USEC_POLL 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define MTK_CQDMA_TIMEOUT_POLL 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define MTK_CQDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define MTK_CQDMA_ALIGN_SIZE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* The default number of virtual channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define MTK_CQDMA_NR_VCHANS 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /* The default number of physical channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define MTK_CQDMA_NR_PCHANS 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* Registers for underlying dma manipulation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define MTK_CQDMA_INT_FLAG 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define MTK_CQDMA_INT_EN 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define MTK_CQDMA_EN 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define MTK_CQDMA_RESET 0xc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define MTK_CQDMA_FLUSH 0x14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define MTK_CQDMA_SRC 0x1c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define MTK_CQDMA_DST 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define MTK_CQDMA_LEN1 0x24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define MTK_CQDMA_LEN2 0x28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define MTK_CQDMA_SRC2 0x60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define MTK_CQDMA_DST2 0x64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* Registers setting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define MTK_CQDMA_EN_BIT BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define MTK_CQDMA_INT_FLAG_BIT BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define MTK_CQDMA_INT_EN_BIT BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define MTK_CQDMA_FLUSH_BIT BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define MTK_CQDMA_WARM_RST_BIT BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define MTK_CQDMA_HARD_RST_BIT BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define MTK_CQDMA_MAX_LEN GENMASK(27, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define MTK_CQDMA_ADDR_LIMIT GENMASK(31, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define MTK_CQDMA_ADDR2_SHFIT (32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * struct mtk_cqdma_vdesc - The struct holding info describing virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * descriptor (CVD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * @vd: An instance for struct virt_dma_desc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * @len: The total data size device wants to move
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * @residue: The remaining data size device will move
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @dest: The destination address device wants to move to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * @src: The source address device wants to move from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * @ch: The pointer to the corresponding dma channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * @node: The lise_head struct to build link-list for VDs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * @parent: The pointer to the parent CVD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct mtk_cqdma_vdesc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct virt_dma_desc vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) size_t residue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) dma_addr_t dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) dma_addr_t src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct dma_chan *ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct mtk_cqdma_vdesc *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * struct mtk_cqdma_pchan - The struct holding info describing physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * channel (PC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * @queue: Queue for the PDs issued to this PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * @base: The mapped register I/O base of this PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * @irq: The IRQ that this PC are using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * @refcnt: Track how many VCs are using this PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * @tasklet: Tasklet for this PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * @lock: Lock protect agaisting multiple VCs access PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct mtk_cqdma_pchan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct list_head queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) u32 irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) refcount_t refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct tasklet_struct tasklet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /* lock to protect PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * struct mtk_cqdma_vchan - The struct holding info describing virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * channel (VC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * @vc: An instance for struct virt_dma_chan
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * @pc: The pointer to the underlying PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * @issue_completion: The wait for all issued descriptors completited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * @issue_synchronize: Bool indicating channel synchronization starts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct mtk_cqdma_vchan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct virt_dma_chan vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct mtk_cqdma_pchan *pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct completion issue_completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) bool issue_synchronize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * struct mtk_cqdma_device - The struct holding info describing CQDMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * @ddev: An instance for struct dma_device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * @clk: The clock that device internal is using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * @dma_requests: The number of VCs the device supports to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * @dma_channels: The number of PCs the device supports to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * @vc: The pointer to all available VCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * @pc: The pointer to all the underlying PCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct mtk_cqdma_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct dma_device ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct clk *clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) u32 dma_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u32 dma_channels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct mtk_cqdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct mtk_cqdma_pchan **pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return container_of(chan->device, struct mtk_cqdma_device, ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return container_of(vd, struct mtk_cqdma_vdesc, vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return cqdma->ddev.dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return readl(pc->base + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) writel_relaxed(val, pc->base + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) u32 mask, u32 set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) val = mtk_dma_read(pc, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) val &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) val |= set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) mtk_dma_write(pc, reg, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) mtk_dma_rmw(pc, reg, 0, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) mtk_dma_rmw(pc, reg, val, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) kfree(to_cqdma_vdesc(vd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) u32 status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (!atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) !(status & MTK_CQDMA_EN_BIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) MTK_CQDMA_USEC_POLL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) MTK_CQDMA_TIMEOUT_POLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) !(status & MTK_CQDMA_EN_BIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) MTK_CQDMA_USEC_POLL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) MTK_CQDMA_TIMEOUT_POLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return mtk_cqdma_poll_engine_done(pc, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct mtk_cqdma_vdesc *cvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /* wait for the previous transaction done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (mtk_cqdma_poll_engine_done(pc, true) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* warm reset the dma engine for the new transaction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (mtk_cqdma_poll_engine_done(pc, true) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /* setup the source */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) /* setup the destination */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) /* setup the length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* start dma engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct virt_dma_desc *vd, *vd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct mtk_cqdma_pchan *pc = cvc->pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct mtk_cqdma_vdesc *cvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) bool trigger_engine = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) lockdep_assert_held(&cvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) lockdep_assert_held(&pc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* need to trigger dma engine if PC's queue is empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (list_empty(&pc->queue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) trigger_engine = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) cvd = to_cqdma_vdesc(vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) /* add VD into PC's queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) list_add_tail(&cvd->node, &pc->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* start the dma engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (trigger_engine)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) mtk_cqdma_start(pc, cvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /* remove VD from list desc_issued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) list_del(&vd->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * return true if this VC is active,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * meaning that there are VDs under processing by the PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct mtk_cqdma_vdesc *cvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) list_for_each_entry(cvd, &cvc->pc->queue, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (cvc == to_cqdma_vchan(cvd->ch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * return the pointer of the CVD that is just consumed by the PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static struct mtk_cqdma_vdesc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) *mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct mtk_cqdma_vchan *cvc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct mtk_cqdma_vdesc *cvd, *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /* consume a CVD from PC's queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) cvd = list_first_entry_or_null(&pc->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct mtk_cqdma_vdesc, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (unlikely(!cvd || !cvd->parent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) cvc = to_cqdma_vchan(cvd->ch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) ret = cvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* update residue of the parent CVD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) cvd->parent->residue -= cvd->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /* delete CVD from PC's queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) list_del(&cvd->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) spin_lock(&cvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /* check whether all the child CVDs completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (!cvd->parent->residue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /* add the parent VD into list desc_completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) vchan_cookie_complete(&cvd->parent->vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) /* setup completion if this VC is under synchronization */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) complete(&cvc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) cvc->issue_synchronize = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) spin_unlock(&cvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /* start transaction for next CVD in the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) cvd = list_first_entry_or_null(&pc->queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct mtk_cqdma_vdesc, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (cvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) mtk_cqdma_start(pc, cvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) struct mtk_cqdma_vdesc *cvd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) spin_lock_irqsave(&pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /* consume the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) cvd = mtk_cqdma_consume_work_queue(pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) spin_unlock_irqrestore(&pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /* submit the next CVD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (cvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) dma_run_dependencies(&cvd->vd.tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * free child CVD after completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * the parent CVD would be freeed with desc_free by user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (cvd->parent != cvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) kfree(cvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /* re-enable interrupt before leaving tasklet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) enable_irq(pc->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct mtk_cqdma_device *cqdma = devid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) irqreturn_t ret = IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) bool schedule_tasklet = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* clear interrupt flags for each PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) spin_lock(&cqdma->pc[i]->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (mtk_dma_read(cqdma->pc[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /* clear interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) MTK_CQDMA_INT_FLAG_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) schedule_tasklet = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) ret = IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) spin_unlock(&cqdma->pc[i]->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (schedule_tasklet) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* disable interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) disable_irq_nosync(cqdma->pc[i]->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /* schedule the tasklet to handle the transactions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) tasklet_schedule(&cqdma->pc[i]->tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) dma_cookie_t cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) spin_lock_irqsave(&cvc->pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) list_for_each_entry(vd, &cvc->pc->queue, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (vd->tx.cookie == cookie) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) spin_unlock_irqrestore(&cvc->pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) spin_unlock_irqrestore(&cvc->pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) list_for_each_entry(vd, &cvc->vc.desc_issued, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (vd->tx.cookie == cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) dma_cookie_t cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) struct dma_tx_state *txstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct mtk_cqdma_vdesc *cvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) enum dma_status ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) size_t bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) ret = dma_cookie_status(c, cookie, txstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (ret == DMA_COMPLETE || !txstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) spin_lock_irqsave(&cvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) vd = mtk_cqdma_find_active_desc(c, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) spin_unlock_irqrestore(&cvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (vd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) cvd = to_cqdma_vdesc(vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) bytes = cvd->residue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) dma_set_residue(txstate, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static void mtk_cqdma_issue_pending(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) unsigned long pc_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) unsigned long vc_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /* acquire PC's lock before VS's lock for lock dependency in tasklet */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) spin_lock_irqsave(&cvc->pc->lock, pc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) spin_lock_irqsave(&cvc->vc.lock, vc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (vchan_issue_pending(&cvc->vc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) mtk_cqdma_issue_vchan_pending(cvc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) dma_addr_t src, size_t len, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct mtk_cqdma_vdesc **cvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) size_t i, tlen, nr_vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * In the case that trsanction length is larger than the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * DMA engine supports, a single memcpy transaction needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * to be separated into several DMA transactions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Each DMA transaction would be described by a CVD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * and the first one is referred as the parent CVD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * while the others are child CVDs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * The parent CVD's tx descriptor is the only tx descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * returned to the DMA user, and it should not be completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * until all the child CVDs completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (!cvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) for (i = 0; i < nr_vd; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!cvd[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) for (; i > 0; --i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) kfree(cvd[i - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* setup dma channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) cvd[i]->ch = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) /* setup sourece, destination, and length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) cvd[i]->len = tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) cvd[i]->src = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) cvd[i]->dest = dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /* setup tx descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) tx->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (!i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) cvd[0]->residue = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) prev_tx->next = tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) cvd[i]->residue = tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) cvd[i]->parent = cvd[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /* update the src, dest, len, prev_tx for the next CVD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) src += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) dest += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) len -= tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) prev_tx = tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return &cvd[0]->vd.tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct virt_dma_chan *vc = to_virt_chan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) LIST_HEAD(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * set desc_allocated, desc_submitted,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * and desc_issued as the candicates to be freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) spin_lock_irqsave(&vc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) list_splice_tail_init(&vc->desc_allocated, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) list_splice_tail_init(&vc->desc_submitted, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) list_splice_tail_init(&vc->desc_issued, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) spin_unlock_irqrestore(&vc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /* free descriptor lists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) vchan_dma_desc_free_list(vc, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) static void mtk_cqdma_free_active_desc(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) bool sync_needed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) unsigned long pc_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) unsigned long vc_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /* acquire PC's lock first due to lock dependency in dma ISR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) spin_lock_irqsave(&cvc->pc->lock, pc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) spin_lock_irqsave(&cvc->vc.lock, vc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* synchronization is required if this VC is active */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (mtk_cqdma_is_vchan_active(cvc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) cvc->issue_synchronize = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) sync_needed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* waiting for the completion of this VC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (sync_needed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) wait_for_completion(&cvc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) /* free all descriptors in list desc_completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) vchan_synchronize(&cvc->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) "Desc pending still in list desc_completed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static int mtk_cqdma_terminate_all(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) /* free descriptors not processed yet by hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) mtk_cqdma_free_inactive_desc(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /* free descriptors being processed by hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) mtk_cqdma_free_active_desc(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct mtk_cqdma_pchan *pc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) u32 i, min_refcnt = U32_MAX, refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /* allocate PC with the minimun refcount */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) for (i = 0; i < cqdma->dma_channels; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) refcnt = refcount_read(&cqdma->pc[i]->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (refcnt < min_refcnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) pc = cqdma->pc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) min_refcnt = refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (!pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) spin_lock_irqsave(&pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (!refcount_read(&pc->refcnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) /* allocate PC when the refcount is zero */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) mtk_cqdma_hard_reset(pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) /* enable interrupt for this PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * refcount_inc would complain increment on 0; use-after-free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * Thus, we need to explicitly set it as 1 initially.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) refcount_set(&pc->refcnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) refcount_inc(&pc->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) spin_unlock_irqrestore(&pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) vc->pc = pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* free all descriptors in all lists on the VC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) mtk_cqdma_terminate_all(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) spin_lock_irqsave(&cvc->pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) /* PC is not freed until there is no VC mapped to it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (refcount_dec_and_test(&cvc->pc->refcnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /* start the flush operation and stop the engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* wait for the completion of flush operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* clear the flush bit and interrupt flag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) MTK_CQDMA_INT_FLAG_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /* disable interrupt for this PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) spin_unlock_irqrestore(&cvc->pc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) pm_runtime_enable(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) pm_runtime_get_sync(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) err = clk_prepare_enable(cqdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) pm_runtime_put_sync(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) pm_runtime_disable(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) /* reset all PCs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) for (i = 0; i < cqdma->dma_channels; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) clk_disable_unprepare(cqdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) pm_runtime_put_sync(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) pm_runtime_disable(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) /* reset all PCs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) for (i = 0; i < cqdma->dma_channels; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) clk_disable_unprepare(cqdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) pm_runtime_put_sync(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) pm_runtime_disable(cqdma2dev(cqdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) static const struct of_device_id mtk_cqdma_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) { .compatible = "mediatek,mt6765-cqdma" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) { /* sentinel */ }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) static int mtk_cqdma_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct mtk_cqdma_device *cqdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) struct mtk_cqdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) struct dma_device *dd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (!cqdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) dd = &cqdma->ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) if (IS_ERR(cqdma->clk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) dev_err(&pdev->dev, "No clock for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) dev_name(&pdev->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) return PTR_ERR(cqdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) dma_cap_set(DMA_MEMCPY, dd->cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) dd->device_tx_status = mtk_cqdma_tx_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) dd->device_issue_pending = mtk_cqdma_issue_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) dd->device_terminate_all = mtk_cqdma_terminate_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) dd->directions = BIT(DMA_MEM_TO_MEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) dd->dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) INIT_LIST_HEAD(&dd->channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) "dma-requests",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) &cqdma->dma_requests)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) dev_info(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) "Using %u as missing dma-requests property\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) MTK_CQDMA_NR_VCHANS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) "dma-channels",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) &cqdma->dma_channels)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) dev_info(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) "Using %u as missing dma-channels property\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) MTK_CQDMA_NR_PCHANS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) sizeof(*cqdma->pc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) if (!cqdma->pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /* initialization for PCs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) for (i = 0; i < cqdma->dma_channels; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) sizeof(**cqdma->pc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (!cqdma->pc[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) INIT_LIST_HEAD(&cqdma->pc[i]->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) spin_lock_init(&cqdma->pc[i]->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) refcount_set(&cqdma->pc[i]->refcnt, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (IS_ERR(cqdma->pc[i]->base))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return PTR_ERR(cqdma->pc[i]->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) /* allocate IRQ resource */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if (!res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) dev_err(&pdev->dev, "No irq resource for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) dev_name(&pdev->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) cqdma->pc[i]->irq = res->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) mtk_cqdma_irq, 0, dev_name(&pdev->dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) cqdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) dev_err(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) "request_irq failed with err %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) /* allocate resource for VCs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) sizeof(*cqdma->vc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (!cqdma->vc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) for (i = 0; i < cqdma->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) vc = &cqdma->vc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) vc->vc.desc_free = mtk_cqdma_vdesc_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) vchan_init(&vc->vc, dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) init_completion(&vc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) err = dma_async_device_register(dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) err = of_dma_controller_register(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) of_dma_xlate_by_chan_id, cqdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) dev_err(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) "MediaTek CQDMA OF registration failed %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) goto err_unregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) err = mtk_cqdma_hw_init(cqdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) dev_err(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) "MediaTek CQDMA HW initialization failed %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) goto err_unregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) platform_set_drvdata(pdev, cqdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /* initialize tasklet for each PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) for (i = 0; i < cqdma->dma_channels; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) err_unregister:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) dma_async_device_unregister(dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) static int mtk_cqdma_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) struct mtk_cqdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) /* kill VC task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) for (i = 0; i < cqdma->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) vc = &cqdma->vc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) list_del(&vc->vc.chan.device_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) tasklet_kill(&vc->vc.task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) /* disable interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) for (i = 0; i < cqdma->dma_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) MTK_CQDMA_INT_EN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) /* Waits for any pending IRQ handlers to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) synchronize_irq(cqdma->pc[i]->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) tasklet_kill(&cqdma->pc[i]->tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) /* disable hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) mtk_cqdma_hw_deinit(cqdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) dma_async_device_unregister(&cqdma->ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) of_dma_controller_free(pdev->dev.of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) static struct platform_driver mtk_cqdma_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) .probe = mtk_cqdma_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) .remove = mtk_cqdma_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) .name = KBUILD_MODNAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) .of_match_table = mtk_cqdma_match,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) module_platform_driver(mtk_cqdma_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) MODULE_LICENSE("GPL v2");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags