^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright 2015 Linaro.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/dmapool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/of_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/of_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "virt-dma.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define DRIVER_NAME "zx-dma"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define DMA_ALIGN 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define DMA_MAX_SIZE (0x10000 - 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define REG_ZX_SRC_ADDR 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define REG_ZX_DST_ADDR 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define REG_ZX_TX_X_COUNT 0x08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define REG_ZX_TX_ZY_COUNT 0x0c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define REG_ZX_SRC_ZY_STEP 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define REG_ZX_DST_ZY_STEP 0x14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define REG_ZX_LLI_ADDR 0x1c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define REG_ZX_CTRL 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define REG_ZX_TC_IRQ 0x800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define REG_ZX_SRC_ERR_IRQ 0x804
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define REG_ZX_DST_ERR_IRQ 0x808
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define REG_ZX_CFG_ERR_IRQ 0x80c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define REG_ZX_TC_IRQ_RAW 0x810
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define REG_ZX_SRC_ERR_IRQ_RAW 0x814
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define REG_ZX_DST_ERR_IRQ_RAW 0x818
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define REG_ZX_STATUS 0x820
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define REG_ZX_DMA_GRP_PRIO 0x824
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define REG_ZX_DMA_ARB 0x828
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define ZX_FORCE_CLOSE BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define ZX_MAX_BURST_LEN 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define ZX_IRQ_ENABLE_ALL (3 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define ZX_DST_FIFO_MODE BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define ZX_SRC_FIFO_MODE BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define ZX_SOFT_REQ BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define ZX_CH_ENABLE BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define ZX_DMA_BUSWIDTHS \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) enum zx_dma_burst_width {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ZX_DMA_WIDTH_8BIT = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ZX_DMA_WIDTH_16BIT = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) ZX_DMA_WIDTH_32BIT = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) ZX_DMA_WIDTH_64BIT = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct zx_desc_hw {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) u32 saddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) u32 daddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) u32 src_x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) u32 src_zy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) u32 src_zy_step;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u32 dst_zy_step;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u32 reserved1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) u32 lli;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) u32 ctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) u32 reserved[7]; /* pack as hardware registers region size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } __aligned(32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct zx_dma_desc_sw {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct virt_dma_desc vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) dma_addr_t desc_hw_lli;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) size_t desc_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct zx_desc_hw *desc_hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct zx_dma_phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct zx_dma_chan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct dma_slave_config slave_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int id; /* Request phy chan id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) u32 ccfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) u32 cyclic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct virt_dma_chan vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct zx_dma_phy *phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) dma_addr_t dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) enum dma_status status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct zx_dma_phy {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct zx_dma_chan *vchan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct zx_dma_desc_sw *ds_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) struct zx_dma_desc_sw *ds_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct zx_dma_dev {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct dma_device slave;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) spinlock_t lock; /* lock for ch and phy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct list_head chan_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct zx_dma_phy *phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct zx_dma_chan *chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct clk *clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct dma_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u32 dma_channels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) u32 dma_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return container_of(chan, struct zx_dma_chan, vc.chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) u32 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) val = readl_relaxed(phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) val &= ~ZX_CH_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) val |= ZX_FORCE_CLOSE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) writel_relaxed(val, phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) val = 0x1 << phy->idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return readl_relaxed(phy->base + REG_ZX_LLI_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return readl_relaxed(d->base + REG_ZX_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void zx_dma_init_state(struct zx_dma_dev *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* set same priority */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /* clear all irq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static int zx_dma_start_txd(struct zx_dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (!c->phy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (vd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct zx_dma_desc_sw *ds =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) container_of(vd, struct zx_dma_desc_sw, vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * fetch and remove request from vc->desc_issued
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * so vc->desc_issued only contains desc pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) list_del(&ds->vd.node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) c->phy->ds_run = ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) c->phy->ds_done = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /* start dma */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) zx_dma_set_desc(c->phy, ds->desc_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) c->phy->ds_done = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) c->phy->ds_run = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static void zx_dma_task(struct zx_dma_dev *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct zx_dma_phy *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct zx_dma_chan *c, *cn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unsigned pch, pch_alloc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) /* check new dma request of running channel in vc->desc_issued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) list_for_each_entry_safe(c, cn, &d->slave.channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) vc.chan.device_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) spin_lock_irqsave(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) p = c->phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (p && p->ds_done && zx_dma_start_txd(c)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /* No current txd associated with this channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Mark this channel free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) c->phy = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) p->vchan = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) spin_unlock_irqrestore(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* check new channel request in d->chan_pending */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) spin_lock_irqsave(&d->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) while (!list_empty(&d->chan_pending)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) c = list_first_entry(&d->chan_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct zx_dma_chan, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) p = &d->phy[c->id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!p->vchan) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /* remove from d->chan_pending */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) list_del_init(&c->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) pch_alloc |= 1 << c->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /* Mark this channel allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) p->vchan = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) c->phy = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) spin_unlock_irqrestore(&d->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) for (pch = 0; pch < d->dma_channels; pch++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (pch_alloc & (1 << pch)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) p = &d->phy[pch];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) c = p->vchan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) spin_lock_irqsave(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) zx_dma_start_txd(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) spin_unlock_irqrestore(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static irqreturn_t zx_dma_int_handler(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct zx_dma_phy *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct zx_dma_chan *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) u32 i, irq_chan = 0, task = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) while (tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) i = __ffs(tc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) tc &= ~BIT(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) p = &d->phy[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) c = p->vchan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) spin_lock(&c->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (c->cyclic) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) vchan_cyclic_callback(&p->ds_run->vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) vchan_cookie_complete(&p->ds_run->vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) p->ds_done = p->ds_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) task = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) spin_unlock(&c->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) irq_chan |= BIT(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (serr || derr || cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) serr, derr, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) zx_dma_task(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static void zx_dma_free_chan_resources(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct zx_dma_dev *d = to_zx_dma(chan->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) spin_lock_irqsave(&d->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) list_del_init(&c->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) spin_unlock_irqrestore(&d->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) vchan_free_chan_resources(&c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) c->ccfg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static enum dma_status zx_dma_tx_status(struct dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) dma_cookie_t cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) struct dma_tx_state *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct zx_dma_phy *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) enum dma_status ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) size_t bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) ret = dma_cookie_status(&c->vc.chan, cookie, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (ret == DMA_COMPLETE || !state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) spin_lock_irqsave(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) p = c->phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) ret = c->status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * If the cookie is on our issue queue, then the residue is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * its total size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) vd = vchan_find_desc(&c->vc, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (vd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) } else if ((!p) || (!p->ds_run)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct zx_dma_desc_sw *ds = p->ds_run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) u32 clli = 0, index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) clli = zx_dma_get_curr_lli(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) index = (clli - ds->desc_hw_lli) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) sizeof(struct zx_desc_hw) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) for (; index < ds->desc_num; index++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) bytes += ds->desc_hw[index].src_x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* end of lli */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (!ds->desc_hw[index].lli)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) spin_unlock_irqrestore(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) dma_set_residue(state, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static void zx_dma_issue_pending(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct zx_dma_dev *d = to_zx_dma(chan->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) int issue = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) spin_lock_irqsave(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* add request to vc->desc_issued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (vchan_issue_pending(&c->vc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) spin_lock(&d->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (!c->phy && list_empty(&c->node)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* if new channel, add chan_pending */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) list_add_tail(&c->node, &d->chan_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) issue = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) spin_unlock(&d->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) spin_unlock_irqrestore(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (issue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) zx_dma_task(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) dma_addr_t src, size_t len, u32 num, u32 ccfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if ((num + 1) < ds->desc_num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) sizeof(struct zx_desc_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ds->desc_hw[num].saddr = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) ds->desc_hw[num].daddr = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) ds->desc_hw[num].src_x = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) ds->desc_hw[num].ctr = ccfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct zx_dma_desc_sw *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) struct zx_dma_dev *d = to_zx_dma(chan->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (num > lli_limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) &c->vc, num, lli_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) ds = kzalloc(sizeof(*ds), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (!ds->desc_hw) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) kfree(ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) ds->desc_num = num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) switch (width) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) case DMA_SLAVE_BUSWIDTH_1_BYTE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) case DMA_SLAVE_BUSWIDTH_2_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) case DMA_SLAVE_BUSWIDTH_4_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) case DMA_SLAVE_BUSWIDTH_8_BYTES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return ffs(width) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) return ZX_DMA_WIDTH_32BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) struct dma_slave_config *cfg = &c->slave_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) enum zx_dma_burst_width src_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) enum zx_dma_burst_width dst_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) u32 maxburst = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) switch (dir) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) case DMA_MEM_TO_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) case DMA_MEM_TO_DEV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) c->dev_addr = cfg->dst_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /* dst len is calculated from src width, len and dst width.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * We need make sure dst len not exceed MAX LEN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * Trailing single transaction that does not fill a full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * burst also require identical src/dst data width.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) dst_width = zx_dma_burst_width(cfg->dst_addr_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) maxburst = cfg->dst_maxburst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) maxburst = maxburst < ZX_MAX_BURST_LEN ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) maxburst : ZX_MAX_BURST_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) | ZX_SRC_BURST_LEN(maxburst - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) | ZX_SRC_BURST_WIDTH(dst_width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) | ZX_DST_BURST_WIDTH(dst_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) case DMA_DEV_TO_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) c->dev_addr = cfg->src_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) src_width = zx_dma_burst_width(cfg->src_addr_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) maxburst = cfg->src_maxburst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) maxburst = maxburst < ZX_MAX_BURST_LEN ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) maxburst : ZX_MAX_BURST_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) | ZX_SRC_BURST_LEN(maxburst - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) | ZX_SRC_BURST_WIDTH(src_width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) | ZX_DST_BURST_WIDTH(src_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static struct dma_async_tx_descriptor *zx_dma_prep_memcpy(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) size_t len, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct zx_dma_desc_sw *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) size_t copy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) int num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (zx_pre_config(c, DMA_MEM_TO_MEM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) ds = zx_alloc_desc_resource(num, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (!ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) ds->size = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) copy = min_t(size_t, len, DMA_MAX_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) src += copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) dst += copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) len -= copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) } while (len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) c->cyclic = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ds->desc_hw[num - 1].lli = 0; /* end of link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) return vchan_tx_prep(&c->vc, &ds->vd, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) enum dma_transfer_direction dir, unsigned long flags, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) struct zx_dma_desc_sw *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) size_t len, avail, total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) dma_addr_t addr, src = 0, dst = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) int num = sglen, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (!sgl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (zx_pre_config(c, dir))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) for_each_sg(sgl, sg, sglen, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) avail = sg_dma_len(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (avail > DMA_MAX_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) ds = zx_alloc_desc_resource(num, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) c->cyclic = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) for_each_sg(sgl, sg, sglen, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) addr = sg_dma_address(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) avail = sg_dma_len(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) total += avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) len = min_t(size_t, avail, DMA_MAX_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (dir == DMA_MEM_TO_DEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) src = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) dst = c->dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) } else if (dir == DMA_DEV_TO_MEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) src = c->dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) dst = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) addr += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) avail -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) } while (avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) ds->desc_hw[num - 1].lli = 0; /* end of link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ds->size = total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return vchan_tx_prep(&c->vc, &ds->vd, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) size_t period_len, enum dma_transfer_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct zx_dma_desc_sw *ds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) dma_addr_t src = 0, dst = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) int num_periods = buf_len / period_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) int buf = 0, num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (period_len > DMA_MAX_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) dev_err(chan->device->dev, "maximum period size exceeded\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (zx_pre_config(c, dir))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) ds = zx_alloc_desc_resource(num_periods, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (!ds)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) c->cyclic = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) while (buf < buf_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (dir == DMA_MEM_TO_DEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) src = dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) dst = c->dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) } else if (dir == DMA_DEV_TO_MEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) src = c->dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) dst = dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) zx_dma_fill_desc(ds, dst, src, period_len, num++,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) c->ccfg | ZX_IRQ_ENABLE_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) dma_addr += period_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) buf += period_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) ds->desc_hw[num - 1].lli = ds->desc_hw_lli;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ds->size = buf_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return vchan_tx_prep(&c->vc, &ds->vd, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static int zx_dma_config(struct dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct dma_slave_config *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (!cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) memcpy(&c->slave_cfg, cfg, sizeof(*cfg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) static int zx_dma_terminate_all(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) struct zx_dma_dev *d = to_zx_dma(chan->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) struct zx_dma_phy *p = c->phy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) LIST_HEAD(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /* Prevent this channel being scheduled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) spin_lock(&d->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) list_del_init(&c->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) spin_unlock(&d->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /* Clear the tx descriptor lists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) spin_lock_irqsave(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) vchan_get_all_descriptors(&c->vc, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /* vchan is assigned to a pchan - stop the channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) zx_dma_terminate_chan(p, d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) c->phy = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) p->vchan = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) p->ds_run = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) p->ds_done = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) spin_unlock_irqrestore(&c->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) vchan_dma_desc_free_list(&c->vc, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static int zx_dma_transfer_pause(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) u32 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) val &= ~ZX_CH_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) static int zx_dma_transfer_resume(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct zx_dma_chan *c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) u32 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) val |= ZX_CH_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static void zx_dma_free_desc(struct virt_dma_desc *vd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) struct zx_dma_desc_sw *ds =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) container_of(vd, struct zx_dma_desc_sw, vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) kfree(ds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) static const struct of_device_id zx6702_dma_dt_ids[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) { .compatible = "zte,zx296702-dma", },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) struct of_dma *ofdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) struct zx_dma_dev *d = ofdma->of_dma_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) unsigned int request = dma_spec->args[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) struct dma_chan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) struct zx_dma_chan *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (request >= d->dma_requests)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) chan = dma_get_any_slave_channel(&d->slave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (!chan) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) dev_err(d->slave.dev, "get channel fail in %s.\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) c = to_zx_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) c->id = request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) c->id, &c->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) static int zx_dma_probe(struct platform_device *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct zx_dma_dev *d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) int i, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (!d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) d->base = devm_platform_ioremap_resource(op, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (IS_ERR(d->base))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) return PTR_ERR(d->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) of_property_read_u32((&op->dev)->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) "dma-channels", &d->dma_channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) of_property_read_u32((&op->dev)->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) "dma-requests", &d->dma_requests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (!d->dma_requests || !d->dma_channels)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) d->clk = devm_clk_get(&op->dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (IS_ERR(d->clk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) dev_err(&op->dev, "no dma clk\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return PTR_ERR(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) d->irq = platform_get_irq(op, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) 0, DRIVER_NAME, d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) /* A DMA memory pool for LLIs, align on 32-byte boundary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) LLI_BLOCK_SIZE, 32, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (!d->pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) /* init phy channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) d->phy = devm_kcalloc(&op->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) d->dma_channels, sizeof(struct zx_dma_phy), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (!d->phy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) for (i = 0; i < d->dma_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) struct zx_dma_phy *p = &d->phy[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) p->idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) p->base = d->base + i * 0x40;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) INIT_LIST_HEAD(&d->slave.channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) dma_cap_set(DMA_PRIVATE, d->slave.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) d->slave.dev = &op->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) d->slave.device_free_chan_resources = zx_dma_free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) d->slave.device_tx_status = zx_dma_tx_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) d->slave.device_issue_pending = zx_dma_issue_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) d->slave.device_config = zx_dma_config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) d->slave.device_terminate_all = zx_dma_terminate_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) d->slave.device_pause = zx_dma_transfer_pause;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) d->slave.device_resume = zx_dma_transfer_resume;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) d->slave.copy_align = DMA_ALIGN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) | BIT(DMA_DEV_TO_MEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) /* init virtual channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) d->chans = devm_kcalloc(&op->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) d->dma_requests, sizeof(struct zx_dma_chan), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) if (!d->chans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) for (i = 0; i < d->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) struct zx_dma_chan *c = &d->chans[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) c->status = DMA_IN_PROGRESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) INIT_LIST_HEAD(&c->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) c->vc.desc_free = zx_dma_free_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) vchan_init(&c->vc, &d->slave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /* Enable clock before accessing registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) ret = clk_prepare_enable(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) goto zx_dma_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) zx_dma_init_state(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) spin_lock_init(&d->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) INIT_LIST_HEAD(&d->chan_pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) platform_set_drvdata(op, d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) ret = dma_async_device_register(&d->slave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) goto clk_dis;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) ret = of_dma_controller_register((&op->dev)->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) zx_of_dma_simple_xlate, d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) goto of_dma_register_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) dev_info(&op->dev, "initialized\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) of_dma_register_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) dma_async_device_unregister(&d->slave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) clk_dis:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) clk_disable_unprepare(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) zx_dma_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static int zx_dma_remove(struct platform_device *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct zx_dma_chan *c, *cn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) struct zx_dma_dev *d = platform_get_drvdata(op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) /* explictly free the irq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) devm_free_irq(&op->dev, d->irq, d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) dma_async_device_unregister(&d->slave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) of_dma_controller_free((&op->dev)->of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) list_for_each_entry_safe(c, cn, &d->slave.channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) vc.chan.device_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) list_del(&c->vc.chan.device_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) clk_disable_unprepare(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) #ifdef CONFIG_PM_SLEEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) static int zx_dma_suspend_dev(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) struct zx_dma_dev *d = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) u32 stat = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) stat = zx_dma_get_chan_stat(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) if (stat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) dev_warn(d->slave.dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) "chan %d is running fail to suspend\n", stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) clk_disable_unprepare(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) static int zx_dma_resume_dev(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) struct zx_dma_dev *d = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) ret = clk_prepare_enable(d->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) zx_dma_init_state(d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) static struct platform_driver zx_pdma_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) .name = DRIVER_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) .pm = &zx_dma_pmops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) .of_match_table = zx6702_dma_dt_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) .probe = zx_dma_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) .remove = zx_dma_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) module_platform_driver(zx_pdma_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) MODULE_AUTHOR("Jun Nie jun.nie@linaro.org");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) MODULE_LICENSE("GPL v2");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags