^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) // Copyright (c) 2017-2018 MediaTek Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Driver for MediaTek High-Speed DMA Controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Sean Wang <sean.wang@mediatek.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/iopoll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/of_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/of_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include "../virt-dma.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define MTK_HSDMA_USEC_POLL 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define MTK_HSDMA_TIMEOUT_POLL 200000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define MTK_HSDMA_DMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* The default number of virtual channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define MTK_HSDMA_NR_VCHANS 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* Only one physical channel supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define MTK_HSDMA_NR_MAX_PCHANS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* Macro for physical descriptor (PD) manipulation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* The number of PD which must be 2 of power */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define MTK_DMA_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define MTK_HSDMA_NEXT_DESP_IDX(x, y) (((x) + 1) & ((y) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define MTK_HSDMA_LAST_DESP_IDX(x, y) (((x) - 1) & ((y) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define MTK_HSDMA_MAX_LEN 0x3f80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define MTK_HSDMA_ALIGN_SIZE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define MTK_HSDMA_PLEN_MASK 0x3fff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define MTK_HSDMA_DESC_PLEN(x) (((x) & MTK_HSDMA_PLEN_MASK) << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define MTK_HSDMA_DESC_PLEN_GET(x) (((x) >> 16) & MTK_HSDMA_PLEN_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /* Registers for underlying ring manipulation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define MTK_HSDMA_TX_BASE 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define MTK_HSDMA_TX_CNT 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define MTK_HSDMA_TX_CPU 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define MTK_HSDMA_TX_DMA 0xc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define MTK_HSDMA_RX_BASE 0x100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define MTK_HSDMA_RX_CNT 0x104
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define MTK_HSDMA_RX_CPU 0x108
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define MTK_HSDMA_RX_DMA 0x10c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* Registers for global setup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define MTK_HSDMA_GLO 0x204
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define MTK_HSDMA_GLO_MULTI_DMA BIT(10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define MTK_HSDMA_TX_WB_DDONE BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define MTK_HSDMA_BURST_64BYTES (0x2 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define MTK_HSDMA_GLO_RX_BUSY BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define MTK_HSDMA_GLO_RX_DMA BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define MTK_HSDMA_GLO_TX_BUSY BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define MTK_HSDMA_GLO_TX_DMA BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define MTK_HSDMA_GLO_DMA (MTK_HSDMA_GLO_TX_DMA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) MTK_HSDMA_GLO_RX_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define MTK_HSDMA_GLO_BUSY (MTK_HSDMA_GLO_RX_BUSY | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) MTK_HSDMA_GLO_TX_BUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define MTK_HSDMA_GLO_DEFAULT (MTK_HSDMA_GLO_TX_DMA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) MTK_HSDMA_GLO_RX_DMA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) MTK_HSDMA_TX_WB_DDONE | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) MTK_HSDMA_BURST_64BYTES | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) MTK_HSDMA_GLO_MULTI_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /* Registers for reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define MTK_HSDMA_RESET 0x208
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define MTK_HSDMA_RST_TX BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define MTK_HSDMA_RST_RX BIT(16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* Registers for interrupt control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define MTK_HSDMA_DLYINT 0x20c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define MTK_HSDMA_RXDLY_INT_EN BIT(15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* Interrupt fires when the pending number's more than the specified */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define MTK_HSDMA_RXMAX_PINT(x) (((x) & 0x7f) << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) /* Interrupt fires when the pending time's more than the specified in 20 us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define MTK_HSDMA_RXMAX_PTIME(x) ((x) & 0x7f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define MTK_HSDMA_DLYINT_DEFAULT (MTK_HSDMA_RXDLY_INT_EN | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) MTK_HSDMA_RXMAX_PINT(20) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) MTK_HSDMA_RXMAX_PTIME(20))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define MTK_HSDMA_INT_STATUS 0x220
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define MTK_HSDMA_INT_ENABLE 0x228
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define MTK_HSDMA_INT_RXDONE BIT(16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) enum mtk_hsdma_vdesc_flag {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) MTK_HSDMA_VDESC_FINISHED = 0x01,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define IS_MTK_HSDMA_VDESC_FINISHED(x) ((x) == MTK_HSDMA_VDESC_FINISHED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * struct mtk_hsdma_pdesc - This is the struct holding info describing physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * descriptor (PD) and its placement must be kept at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * 4-bytes alignment in little endian order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * @desc1: | The control pad used to indicate hardware how to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * @desc2: | deal with the descriptor such as source and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * @desc3: | destination address and data length. The maximum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * @desc4: | data length each pdesc can handle is 0x3f80 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct mtk_hsdma_pdesc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) __le32 desc1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) __le32 desc2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __le32 desc3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) __le32 desc4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) } __packed __aligned(4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * struct mtk_hsdma_vdesc - This is the struct holding info describing virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * descriptor (VD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * @vd: An instance for struct virt_dma_desc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * @len: The total data size device wants to move
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * @residue: The remaining data size device will move
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * @dest: The destination address device wants to move to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * @src: The source address device wants to move from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct mtk_hsdma_vdesc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct virt_dma_desc vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) size_t residue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) dma_addr_t dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) dma_addr_t src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * struct mtk_hsdma_cb - This is the struct holding extra info required for RX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * ring to know what relevant VD the the PD is being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * mapped to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * @vd: Pointer to the relevant VD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * @flag: Flag indicating what action should be taken when VD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * is completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct mtk_hsdma_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) enum mtk_hsdma_vdesc_flag flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * struct mtk_hsdma_ring - This struct holds info describing underlying ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * @txd: The descriptor TX ring which describes DMA source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * @rxd: The descriptor RX ring which describes DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * destination information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * @cb: The extra information pointed at by RX ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * @tphys: The physical addr of TX ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * @rphys: The physical addr of RX ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * @cur_tptr: Pointer to the next free descriptor used by the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * @cur_rptr: Pointer to the last done descriptor by the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct mtk_hsdma_ring {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct mtk_hsdma_pdesc *txd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct mtk_hsdma_pdesc *rxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct mtk_hsdma_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) dma_addr_t tphys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) dma_addr_t rphys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) u16 cur_tptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) u16 cur_rptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * struct mtk_hsdma_pchan - This is the struct holding info describing physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * channel (PC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * @ring: An instance for the underlying ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * @sz_ring: Total size allocated for the ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * @nr_free: Total number of free rooms in the ring. It would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * be accessed and updated frequently between IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * context and user context to reflect whether ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * can accept requests from VD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct mtk_hsdma_pchan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct mtk_hsdma_ring ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) size_t sz_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) atomic_t nr_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * struct mtk_hsdma_vchan - This is the struct holding info describing virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * channel (VC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * @vc: An instance for struct virt_dma_chan
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * @issue_completion: The wait for all issued descriptors completited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * @issue_synchronize: Bool indicating channel synchronization starts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * @desc_hw_processing: List those descriptors the hardware is processing,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * which is protected by vc.lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct mtk_hsdma_vchan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct virt_dma_chan vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct completion issue_completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) bool issue_synchronize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct list_head desc_hw_processing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * struct mtk_hsdma_soc - This is the struct holding differences among SoCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * @ddone: Bit mask for DDONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * @ls0: Bit mask for LS0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct mtk_hsdma_soc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) __le32 ddone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) __le32 ls0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * struct mtk_hsdma_device - This is the struct holding info describing HSDMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * @ddev: An instance for struct dma_device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * @base: The mapped register I/O base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * @clk: The clock that device internal is using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * @irq: The IRQ that device are using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * @dma_requests: The number of VCs the device supports to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * @vc: The pointer to all available VCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * @pc: The pointer to the underlying PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * @pc_refcnt: Track how many VCs are using the PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * @lock: Lock protect agaisting multiple VCs access PC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * @soc: The pointer to area holding differences among
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * vaious platform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct mtk_hsdma_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct dma_device ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct clk *clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) u32 irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) u32 dma_requests;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) struct mtk_hsdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) struct mtk_hsdma_pchan *pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) refcount_t pc_refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /* Lock used to protect against multiple VCs access PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) const struct mtk_hsdma_soc *soc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static struct mtk_hsdma_device *to_hsdma_dev(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return container_of(chan->device, struct mtk_hsdma_device, ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static inline struct mtk_hsdma_vchan *to_hsdma_vchan(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return container_of(chan, struct mtk_hsdma_vchan, vc.chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) static struct mtk_hsdma_vdesc *to_hsdma_vdesc(struct virt_dma_desc *vd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return container_of(vd, struct mtk_hsdma_vdesc, vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return hsdma->ddev.dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return readl(hsdma->base + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) writel(val, hsdma->base + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) u32 mask, u32 set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) val = mtk_dma_read(hsdma, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) val &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) val |= set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) mtk_dma_write(hsdma, reg, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) mtk_dma_rmw(hsdma, reg, 0, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) mtk_dma_rmw(hsdma, reg, val, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static void mtk_hsdma_vdesc_free(struct virt_dma_desc *vd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) kfree(container_of(vd, struct mtk_hsdma_vdesc, vd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) u32 status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) !(status & MTK_HSDMA_GLO_BUSY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) MTK_HSDMA_USEC_POLL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) MTK_HSDMA_TIMEOUT_POLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct mtk_hsdma_pchan *pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct mtk_hsdma_ring *ring = &pc->ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) memset(pc, 0, sizeof(*pc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * Allocate ring space where [0 ... MTK_DMA_SIZE - 1] is for TX ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * and [MTK_DMA_SIZE ... 2 * MTK_DMA_SIZE - 1] is for RX ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) pc->sz_ring = 2 * MTK_DMA_SIZE * sizeof(*ring->txd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) ring->txd = dma_alloc_coherent(hsdma2dev(hsdma), pc->sz_ring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) &ring->tphys, GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (!ring->txd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) ring->rxd = &ring->txd[MTK_DMA_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ring->rphys = ring->tphys + MTK_DMA_SIZE * sizeof(*ring->txd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) ring->cur_tptr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) ring->cur_rptr = MTK_DMA_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) ring->cb = kcalloc(MTK_DMA_SIZE, sizeof(*ring->cb), GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (!ring->cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) goto err_free_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) atomic_set(&pc->nr_free, MTK_DMA_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /* Disable HSDMA and wait for the completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) err = mtk_hsdma_busy_wait(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) goto err_free_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) /* Reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) mtk_dma_set(hsdma, MTK_HSDMA_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) mtk_dma_clr(hsdma, MTK_HSDMA_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) MTK_HSDMA_RST_TX | MTK_HSDMA_RST_RX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* Setup HSDMA initial pointer in the ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /* Enable HSDMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /* Setup delayed interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /* Enable interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) err_free_cb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) kfree(ring->cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) err_free_dma:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) dma_free_coherent(hsdma2dev(hsdma),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) pc->sz_ring, ring->txd, ring->tphys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) struct mtk_hsdma_pchan *pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) struct mtk_hsdma_ring *ring = &pc->ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* Disable HSDMA and then wait for the completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) mtk_hsdma_busy_wait(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) /* Reset pointer in the ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) kfree(ring->cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) dma_free_coherent(hsdma2dev(hsdma),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) pc->sz_ring, ring->txd, ring->tphys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct mtk_hsdma_pchan *pc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct mtk_hsdma_vdesc *hvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct mtk_hsdma_ring *ring = &pc->ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct mtk_hsdma_pdesc *txd, *rxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) u16 reserved, prev, tlen, num_sgs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /* Protect against PC is accessed by multiple VCs simultaneously */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) spin_lock_irqsave(&hsdma->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Reserve rooms, where pc->nr_free is used to track how many free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * rooms in the ring being updated in user and IRQ context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) num_sgs = DIV_ROUND_UP(hvd->len, MTK_HSDMA_MAX_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) reserved = min_t(u16, num_sgs, atomic_read(&pc->nr_free));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!reserved) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) spin_unlock_irqrestore(&hsdma->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) atomic_sub(reserved, &pc->nr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) while (reserved--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /* Limit size by PD capability for valid data moving */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) tlen = (hvd->len > MTK_HSDMA_MAX_LEN) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) MTK_HSDMA_MAX_LEN : hvd->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * Setup PDs using the remaining VD info mapped on those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * reserved rooms. And since RXD is shared memory between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * host and the device allocated by dma_alloc_coherent call,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * the helper macro WRITE_ONCE can ensure the data written to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * RAM would really happens.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) txd = &ring->txd[ring->cur_tptr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) WRITE_ONCE(txd->desc1, hvd->src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) WRITE_ONCE(txd->desc2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) rxd = &ring->rxd[ring->cur_tptr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) WRITE_ONCE(rxd->desc1, hvd->dest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) WRITE_ONCE(rxd->desc2, MTK_HSDMA_DESC_PLEN(tlen));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /* Associate VD, the PD belonged to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) ring->cb[ring->cur_tptr].vd = &hvd->vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /* Move forward the pointer of TX ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) ring->cur_tptr = MTK_HSDMA_NEXT_DESP_IDX(ring->cur_tptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) MTK_DMA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* Update VD with remaining data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) hvd->src += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) hvd->dest += tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) hvd->len -= tlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * Tagging flag for the last PD for VD will be responsible for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * completing VD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (!hvd->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) prev = MTK_HSDMA_LAST_DESP_IDX(ring->cur_tptr, MTK_DMA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ring->cb[prev].flag = MTK_HSDMA_VDESC_FINISHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) /* Ensure all changes indeed done before we're going on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * Updating into hardware the pointer of TX ring lets HSDMA to take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * action for those pending PDs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) spin_unlock_irqrestore(&hsdma->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct mtk_hsdma_vchan *hvc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct virt_dma_desc *vd, *vd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) lockdep_assert_held(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) list_for_each_entry_safe(vd, vd2, &hvc->vc.desc_issued, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct mtk_hsdma_vdesc *hvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) hvd = to_hsdma_vdesc(vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /* Map VD into PC and all VCs shares a single PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * Move VD from desc_issued to desc_hw_processing when entire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * VD is fit into available PDs. Otherwise, the uncompleted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * VDs would stay in list desc_issued and then restart the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * processing as soon as possible once underlying ring space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * got freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (err == -ENOSPC || hvd->len > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * The extra list desc_hw_processing is used because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * hardware can't provide sufficient information allowing us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * to know what VDs are still working on the underlying ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * Through the additional list, it can help us to implement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * terminate_all, residue calculation and such thing needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * to know detail descriptor status on the hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) list_move_tail(&vd->node, &hvc->desc_hw_processing);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct mtk_hsdma_vchan *hvc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct mtk_hsdma_pdesc *rxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct mtk_hsdma_vdesc *hvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct mtk_hsdma_pchan *pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct mtk_hsdma_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) int i = MTK_DMA_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) __le32 desc2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) u16 next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) /* Read IRQ status */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (unlikely(!(status & MTK_HSDMA_INT_RXDONE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) goto rx_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) pc = hsdma->pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * Using a fail-safe loop with iterations of up to MTK_DMA_SIZE to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * reclaim these finished descriptors: The most number of PDs the ISR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * can handle at one time shouldn't be more than MTK_DMA_SIZE so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * take it as limited count instead of just using a dangerous infinite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * poll.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) while (i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) next = MTK_HSDMA_NEXT_DESP_IDX(pc->ring.cur_rptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) MTK_DMA_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) rxd = &pc->ring.rxd[next];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * If MTK_HSDMA_DESC_DDONE is no specified, that means data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * moving for the PD is still under going.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) desc2 = READ_ONCE(rxd->desc2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (!(desc2 & hsdma->soc->ddone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) cb = &pc->ring.cb[next];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (unlikely(!cb->vd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /* Update residue of VD the associated PD belonged to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) hvd = to_hsdma_vdesc(cb->vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) hvd->residue -= MTK_HSDMA_DESC_PLEN_GET(rxd->desc2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* Complete VD until the relevant last PD is finished */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (IS_MTK_HSDMA_VDESC_FINISHED(cb->flag)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) hvc = to_hsdma_vchan(cb->vd->tx.chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) spin_lock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /* Remove VD from list desc_hw_processing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) list_del(&cb->vd->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* Add VD into list desc_completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) vchan_cookie_complete(cb->vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (hvc->issue_synchronize &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) list_empty(&hvc->desc_hw_processing)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) complete(&hvc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) hvc->issue_synchronize = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) spin_unlock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) cb->flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) cb->vd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * Recycle the RXD with the helper WRITE_ONCE that can ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * data written into RAM would really happens.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) WRITE_ONCE(rxd->desc1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) WRITE_ONCE(rxd->desc2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) pc->ring.cur_rptr = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* Release rooms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) atomic_inc(&pc->nr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /* Ensure all changes indeed done before we're going on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /* Update CPU pointer for those completed PDs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * Acking the pending IRQ allows hardware no longer to keep the used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * IRQ line in certain trigger state when software has completed all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * the finished physical descriptors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (atomic_read(&pc->nr_free) >= MTK_DMA_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /* ASAP handles pending VDs in all VCs after freeing some rooms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) for (i = 0; i < hsdma->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) hvc = &hsdma->vc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) spin_lock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) mtk_hsdma_issue_vchan_pending(hsdma, hvc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) spin_unlock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) rx_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /* All completed PDs are cleaned up, so enable interrupt again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) static irqreturn_t mtk_hsdma_irq(int irq, void *devid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct mtk_hsdma_device *hsdma = devid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * Disable interrupt until all completed PDs are cleaned up in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * mtk_hsdma_free_rooms call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) mtk_hsdma_free_rooms_in_ring(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) static struct virt_dma_desc *mtk_hsdma_find_active_desc(struct dma_chan *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) dma_cookie_t cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) list_for_each_entry(vd, &hvc->desc_hw_processing, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) if (vd->tx.cookie == cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) return vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) list_for_each_entry(vd, &hvc->vc.desc_issued, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (vd->tx.cookie == cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) static enum dma_status mtk_hsdma_tx_status(struct dma_chan *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) dma_cookie_t cookie,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) struct dma_tx_state *txstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) struct mtk_hsdma_vdesc *hvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct virt_dma_desc *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) enum dma_status ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) size_t bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) ret = dma_cookie_status(c, cookie, txstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) if (ret == DMA_COMPLETE || !txstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) spin_lock_irqsave(&hvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) vd = mtk_hsdma_find_active_desc(c, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) spin_unlock_irqrestore(&hvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (vd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) hvd = to_hsdma_vdesc(vd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) bytes = hvd->residue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) dma_set_residue(txstate, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) static void mtk_hsdma_issue_pending(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) spin_lock_irqsave(&hvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (vchan_issue_pending(&hvc->vc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) mtk_hsdma_issue_vchan_pending(hsdma, hvc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) spin_unlock_irqrestore(&hvc->vc.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) static struct dma_async_tx_descriptor *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) mtk_hsdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) dma_addr_t src, size_t len, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) struct mtk_hsdma_vdesc *hvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) hvd = kzalloc(sizeof(*hvd), GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (!hvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) hvd->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) hvd->residue = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) hvd->src = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) hvd->dest = dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return vchan_tx_prep(to_virt_chan(c), &hvd->vd, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) static int mtk_hsdma_free_inactive_desc(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) struct virt_dma_chan *vc = to_virt_chan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) LIST_HEAD(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) spin_lock_irqsave(&vc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) list_splice_tail_init(&vc->desc_allocated, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) list_splice_tail_init(&vc->desc_submitted, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) list_splice_tail_init(&vc->desc_issued, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) spin_unlock_irqrestore(&vc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) /* At the point, we don't expect users put descriptor into VC again */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) vchan_dma_desc_free_list(vc, &head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static void mtk_hsdma_free_active_desc(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct mtk_hsdma_vchan *hvc = to_hsdma_vchan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) bool sync_needed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * Once issue_synchronize is being set, which means once the hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * consumes all descriptors for the channel in the ring, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * synchronization must be be notified immediately it is completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) spin_lock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) if (!list_empty(&hvc->desc_hw_processing)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) hvc->issue_synchronize = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) sync_needed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) spin_unlock(&hvc->vc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (sync_needed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) wait_for_completion(&hvc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * At the point, we expect that all remaining descriptors in the ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * for the channel should be all processing done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) WARN_ONCE(!list_empty(&hvc->desc_hw_processing),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) "Desc pending still in list desc_hw_processing\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) /* Free all descriptors in list desc_completed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) vchan_synchronize(&hvc->vc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) WARN_ONCE(!list_empty(&hvc->vc.desc_completed),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) "Desc pending still in list desc_completed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static int mtk_hsdma_terminate_all(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * Free pending descriptors not processed yet by hardware that have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * previously been submitted to the channel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) mtk_hsdma_free_inactive_desc(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * However, the DMA engine doesn't provide any way to stop these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * descriptors being processed currently by hardware. The only way is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * to just waiting until these descriptors are all processed completely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * through mtk_hsdma_free_active_desc call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) mtk_hsdma_free_active_desc(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) static int mtk_hsdma_alloc_chan_resources(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * Since HSDMA has only one PC, the resource for PC is being allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * when the first VC is being created and the other VCs would run on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * the same PC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (!refcount_read(&hsdma->pc_refcnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * refcount_inc would complain increment on 0; use-after-free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) * Thus, we need to explicitly set it as 1 initially.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) refcount_set(&hsdma->pc_refcnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) refcount_inc(&hsdma->pc_refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static void mtk_hsdma_free_chan_resources(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /* Free all descriptors in all lists on the VC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) mtk_hsdma_terminate_all(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /* The resource for PC is not freed until all the VCs are destroyed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (!refcount_dec_and_test(&hsdma->pc_refcnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) mtk_hsdma_free_pchan(hsdma, hsdma->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) pm_runtime_enable(hsdma2dev(hsdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) pm_runtime_get_sync(hsdma2dev(hsdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) err = clk_prepare_enable(hsdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) clk_disable_unprepare(hsdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) pm_runtime_put_sync(hsdma2dev(hsdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) pm_runtime_disable(hsdma2dev(hsdma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) static const struct mtk_hsdma_soc mt7623_soc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) .ddone = BIT(31),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) .ls0 = BIT(30),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) static const struct mtk_hsdma_soc mt7622_soc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) .ddone = BIT(15),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) .ls0 = BIT(14),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) static const struct of_device_id mtk_hsdma_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) { .compatible = "mediatek,mt7623-hsdma", .data = &mt7623_soc},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) { .compatible = "mediatek,mt7622-hsdma", .data = &mt7622_soc},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) { /* sentinel */ }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) MODULE_DEVICE_TABLE(of, mtk_hsdma_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) static int mtk_hsdma_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) struct mtk_hsdma_device *hsdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) struct mtk_hsdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) struct dma_device *dd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) int i, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (!hsdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) dd = &hsdma->ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) hsdma->base = devm_ioremap_resource(&pdev->dev, res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) if (IS_ERR(hsdma->base))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) return PTR_ERR(hsdma->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) hsdma->soc = of_device_get_match_data(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) if (!hsdma->soc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) dev_err(&pdev->dev, "No device match found\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (IS_ERR(hsdma->clk)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) dev_err(&pdev->dev, "No clock for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) dev_name(&pdev->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return PTR_ERR(hsdma->clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) if (!res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) dev_err(&pdev->dev, "No irq resource for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) dev_name(&pdev->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) hsdma->irq = res->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) refcount_set(&hsdma->pc_refcnt, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) spin_lock_init(&hsdma->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) dma_cap_set(DMA_MEMCPY, dd->cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) dd->copy_align = MTK_HSDMA_ALIGN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) dd->device_alloc_chan_resources = mtk_hsdma_alloc_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) dd->device_free_chan_resources = mtk_hsdma_free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) dd->device_tx_status = mtk_hsdma_tx_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) dd->device_issue_pending = mtk_hsdma_issue_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) dd->device_prep_dma_memcpy = mtk_hsdma_prep_dma_memcpy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) dd->device_terminate_all = mtk_hsdma_terminate_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) dd->src_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) dd->dst_addr_widths = MTK_HSDMA_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) dd->directions = BIT(DMA_MEM_TO_MEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) dd->dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) INIT_LIST_HEAD(&dd->channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) "dma-requests",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) &hsdma->dma_requests)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) dev_info(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) "Using %u as missing dma-requests property\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) MTK_HSDMA_NR_VCHANS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) sizeof(*hsdma->pc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) if (!hsdma->pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) sizeof(*hsdma->vc), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (!hsdma->vc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) for (i = 0; i < hsdma->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) vc = &hsdma->vc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) vc->vc.desc_free = mtk_hsdma_vdesc_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) vchan_init(&vc->vc, dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) init_completion(&vc->issue_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) INIT_LIST_HEAD(&vc->desc_hw_processing);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) err = dma_async_device_register(dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) err = of_dma_controller_register(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) of_dma_xlate_by_chan_id, hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) dev_err(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) "MediaTek HSDMA OF registration failed %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) goto err_unregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) mtk_hsdma_hw_init(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) err = devm_request_irq(&pdev->dev, hsdma->irq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) mtk_hsdma_irq, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) dev_name(&pdev->dev), hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) dev_err(&pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) "request_irq failed with err %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) goto err_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) platform_set_drvdata(pdev, hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) dev_info(&pdev->dev, "MediaTek HSDMA driver registered\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) err_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) mtk_hsdma_hw_deinit(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) of_dma_controller_free(pdev->dev.of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) err_unregister:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) dma_async_device_unregister(dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) static int mtk_hsdma_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct mtk_hsdma_vchan *vc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /* Kill VC task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) for (i = 0; i < hsdma->dma_requests; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) vc = &hsdma->vc[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) list_del(&vc->vc.chan.device_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) tasklet_kill(&vc->vc.task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) /* Disable DMA interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) /* Waits for any pending IRQ handlers to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) synchronize_irq(hsdma->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) /* Disable hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) mtk_hsdma_hw_deinit(hsdma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) dma_async_device_unregister(&hsdma->ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) of_dma_controller_free(pdev->dev.of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static struct platform_driver mtk_hsdma_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) .probe = mtk_hsdma_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) .remove = mtk_hsdma_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) .name = KBUILD_MODNAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) .of_match_table = mtk_hsdma_match,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) module_platform_driver(mtk_hsdma_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) MODULE_DESCRIPTION("MediaTek High-Speed DMA Controller Driver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) MODULE_LICENSE("GPL v2");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags