^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * DMA driver for Xilinx ZynqMP DMA Engine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/dmapool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/dma/xilinx_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/of_address.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/of_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/of_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/of_platform.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/io-64-nonatomic-lo-hi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "../dmaengine.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* Register Offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define ZYNQMP_DMA_ISR 0x100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define ZYNQMP_DMA_IMR 0x104
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define ZYNQMP_DMA_IER 0x108
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define ZYNQMP_DMA_IDS 0x10C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define ZYNQMP_DMA_CTRL0 0x110
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define ZYNQMP_DMA_CTRL1 0x114
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define ZYNQMP_DMA_DATA_ATTR 0x120
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define ZYNQMP_DMA_DSCR_ATTR 0x124
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define ZYNQMP_DMA_SRC_DSCR_WRD0 0x128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define ZYNQMP_DMA_SRC_DSCR_WRD1 0x12C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define ZYNQMP_DMA_SRC_DSCR_WRD2 0x130
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define ZYNQMP_DMA_SRC_DSCR_WRD3 0x134
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define ZYNQMP_DMA_DST_DSCR_WRD0 0x138
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define ZYNQMP_DMA_DST_DSCR_WRD1 0x13C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define ZYNQMP_DMA_DST_DSCR_WRD2 0x140
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define ZYNQMP_DMA_DST_DSCR_WRD3 0x144
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define ZYNQMP_DMA_SRC_START_LSB 0x158
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define ZYNQMP_DMA_SRC_START_MSB 0x15C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define ZYNQMP_DMA_DST_START_LSB 0x160
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define ZYNQMP_DMA_DST_START_MSB 0x164
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define ZYNQMP_DMA_TOTAL_BYTE 0x188
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define ZYNQMP_DMA_RATE_CTRL 0x18C
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define ZYNQMP_DMA_IRQ_SRC_ACCT 0x190
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define ZYNQMP_DMA_IRQ_DST_ACCT 0x194
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define ZYNQMP_DMA_CTRL2 0x200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* Interrupt registers bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define ZYNQMP_DMA_DONE BIT(10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define ZYNQMP_DMA_AXI_WR_DATA BIT(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define ZYNQMP_DMA_AXI_RD_DATA BIT(8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define ZYNQMP_DMA_AXI_RD_DST_DSCR BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define ZYNQMP_DMA_AXI_RD_SRC_DSCR BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define ZYNQMP_DMA_IRQ_DST_ACCT_ERR BIT(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define ZYNQMP_DMA_IRQ_SRC_ACCT_ERR BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define ZYNQMP_DMA_BYTE_CNT_OVRFL BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define ZYNQMP_DMA_DST_DSCR_DONE BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define ZYNQMP_DMA_INV_APB BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* Control 0 register bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define ZYNQMP_DMA_OVR_FETCH BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define ZYNQMP_DMA_POINT_TYPE_SG BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define ZYNQMP_DMA_RATE_CTRL_EN BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* Control 1 register bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define ZYNQMP_DMA_SRC_ISSUE GENMASK(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* Data Attribute register bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define ZYNQMP_DMA_ARBURST GENMASK(27, 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define ZYNQMP_DMA_ARCACHE GENMASK(25, 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define ZYNQMP_DMA_ARCACHE_OFST 22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define ZYNQMP_DMA_ARQOS GENMASK(21, 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define ZYNQMP_DMA_ARQOS_OFST 18
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define ZYNQMP_DMA_ARLEN GENMASK(17, 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define ZYNQMP_DMA_ARLEN_OFST 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define ZYNQMP_DMA_AWBURST GENMASK(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define ZYNQMP_DMA_AWCACHE GENMASK(11, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define ZYNQMP_DMA_AWCACHE_OFST 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define ZYNQMP_DMA_AWQOS GENMASK(7, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define ZYNQMP_DMA_AWQOS_OFST 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define ZYNQMP_DMA_AWLEN GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define ZYNQMP_DMA_AWLEN_OFST 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /* Descriptor Attribute register bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define ZYNQMP_DMA_AXCOHRNT BIT(8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define ZYNQMP_DMA_AXCACHE GENMASK(7, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define ZYNQMP_DMA_AXCACHE_OFST 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define ZYNQMP_DMA_AXQOS GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define ZYNQMP_DMA_AXQOS_OFST 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /* Control register 2 bit field definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define ZYNQMP_DMA_ENABLE BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* Buffer Descriptor definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define ZYNQMP_DMA_DESC_CTRL_STOP 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define ZYNQMP_DMA_DESC_CTRL_COMP_INT 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define ZYNQMP_DMA_DESC_CTRL_SIZE_256 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define ZYNQMP_DMA_DESC_CTRL_COHRNT 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* Interrupt Mask specific definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define ZYNQMP_DMA_INT_ERR (ZYNQMP_DMA_AXI_RD_DATA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ZYNQMP_DMA_AXI_WR_DATA | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) ZYNQMP_DMA_AXI_RD_DST_DSCR | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) ZYNQMP_DMA_AXI_RD_SRC_DSCR | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ZYNQMP_DMA_INV_APB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define ZYNQMP_DMA_INT_OVRFL (ZYNQMP_DMA_BYTE_CNT_OVRFL | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) ZYNQMP_DMA_IRQ_SRC_ACCT_ERR | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) ZYNQMP_DMA_IRQ_DST_ACCT_ERR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define ZYNQMP_DMA_INT_DONE (ZYNQMP_DMA_DONE | ZYNQMP_DMA_DST_DSCR_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define ZYNQMP_DMA_INT_EN_DEFAULT_MASK (ZYNQMP_DMA_INT_DONE | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) ZYNQMP_DMA_INT_ERR | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ZYNQMP_DMA_INT_OVRFL | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ZYNQMP_DMA_DST_DSCR_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* Max number of descriptors per channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define ZYNQMP_DMA_NUM_DESCS 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* Max transfer size per descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define ZYNQMP_DMA_MAX_TRANS_LEN 0x40000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) /* Max burst lengths */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define ZYNQMP_DMA_MAX_DST_BURST_LEN 32768U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define ZYNQMP_DMA_MAX_SRC_BURST_LEN 32768U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /* Reset values for data attributes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define ZYNQMP_DMA_AXCACHE_VAL 0xF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define ZYNQMP_DMA_SRC_ISSUE_RST_VAL 0x1F
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define ZYNQMP_DMA_IDS_DEFAULT_MASK 0xFFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Bus width in bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define ZYNQMP_DMA_BUS_WIDTH_64 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define ZYNQMP_DMA_BUS_WIDTH_128 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define ZDMA_PM_TIMEOUT 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define ZYNQMP_DMA_DESC_SIZE(chan) (chan->desc_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define to_chan(chan) container_of(chan, struct zynqmp_dma_chan, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) common)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) async_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * struct zynqmp_dma_desc_ll - Hw linked list descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * @addr: Buffer address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * @size: Size of the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * @ctrl: Control word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * @nxtdscraddr: Next descriptor base address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * @rsvd: Reserved field and for Hw internal use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct zynqmp_dma_desc_ll {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) u32 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) u32 ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) u64 nxtdscraddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) u64 rsvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * struct zynqmp_dma_desc_sw - Per Transaction structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * @src: Source address for simple mode dma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * @dst: Destination address for simple mode dma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * @len: Transfer length for simple mode dma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * @node: Node in the channel descriptor list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * @tx_list: List head for the current transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * @async_tx: Async transaction descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * @src_v: Virtual address of the src descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * @src_p: Physical address of the src descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * @dst_v: Virtual address of the dst descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * @dst_p: Physical address of the dst descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) struct zynqmp_dma_desc_sw {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) u64 src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u64 dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) u32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct list_head tx_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct dma_async_tx_descriptor async_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct zynqmp_dma_desc_ll *src_v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) dma_addr_t src_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct zynqmp_dma_desc_ll *dst_v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) dma_addr_t dst_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * struct zynqmp_dma_chan - Driver specific DMA channel structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * @zdev: Driver specific device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * @regs: Control registers offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * @lock: Descriptor operation lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * @pending_list: Descriptors waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * @free_list: Descriptors free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * @active_list: Descriptors active
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * @sw_desc_pool: SW descriptor pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * @done_list: Complete descriptors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * @common: DMA common channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * @desc_pool_v: Statically allocated descriptor base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * @desc_pool_p: Physical allocated descriptor base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * @desc_free_cnt: Descriptor available count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * @dev: The dma device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * @irq: Channel IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * @is_dmacoherent: Tells whether dma operations are coherent or not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * @tasklet: Cleanup work after irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * @idle : Channel status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * @desc_size: Size of the low level descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * @err: Channel has errors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * @bus_width: Bus width
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * @src_burst_len: Source burst length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * @dst_burst_len: Dest burst length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct zynqmp_dma_chan {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct zynqmp_dma_device *zdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) void __iomem *regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct list_head pending_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct list_head free_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct list_head active_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct zynqmp_dma_desc_sw *sw_desc_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct list_head done_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) struct dma_chan common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) void *desc_pool_v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) dma_addr_t desc_pool_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) u32 desc_free_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) bool is_dmacoherent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct tasklet_struct tasklet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) bool idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) u32 desc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) bool err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) u32 bus_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) u32 src_burst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) u32 dst_burst_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * struct zynqmp_dma_device - DMA device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * @dev: Device Structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * @common: DMA device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * @chan: Driver specific DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * @clk_main: Pointer to main clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * @clk_apb: Pointer to apb clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct zynqmp_dma_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct dma_device common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) struct zynqmp_dma_chan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct clk *clk_main;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct clk *clk_apb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) lo_hi_writeq(value, chan->regs + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * zynqmp_dma_update_desc_to_ctrlr - Updates descriptor to the controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * @chan: ZynqMP DMA DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * @desc: Transaction descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct zynqmp_dma_desc_sw *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) dma_addr_t addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) addr = desc->src_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) addr = desc->dst_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * zynqmp_dma_desc_config_eod - Mark the descriptor as end descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * @desc: Hw descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) void *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct zynqmp_dma_desc_ll *hw = (struct zynqmp_dma_desc_ll *)desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) hw++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) hw->ctrl |= ZYNQMP_DMA_DESC_CTRL_COMP_INT | ZYNQMP_DMA_DESC_CTRL_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * zynqmp_dma_config_sg_ll_desc - Configure the linked list descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * @sdesc: Hw descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * @src: Source buffer address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * @dst: Destination buffer address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * @len: Transfer length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * @prev: Previous hw descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) struct zynqmp_dma_desc_ll *sdesc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) dma_addr_t src, dma_addr_t dst, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct zynqmp_dma_desc_ll *prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct zynqmp_dma_desc_ll *ddesc = sdesc + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) sdesc->size = ddesc->size = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) sdesc->addr = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) ddesc->addr = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) sdesc->ctrl = ddesc->ctrl = ZYNQMP_DMA_DESC_CTRL_SIZE_256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (chan->is_dmacoherent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) sdesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) ddesc->ctrl |= ZYNQMP_DMA_DESC_CTRL_COHRNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (prev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) dma_addr_t addr = chan->desc_pool_p +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) ddesc = prev + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) prev->nxtdscraddr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * zynqmp_dma_init - Initialize the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static void zynqmp_dma_init(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) val = readl(chan->regs + ZYNQMP_DMA_ISR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) writel(val, chan->regs + ZYNQMP_DMA_ISR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (chan->is_dmacoherent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) val = ZYNQMP_DMA_AXCOHRNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) val = (val & ~ZYNQMP_DMA_AXCACHE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AXCACHE_OFST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (chan->is_dmacoherent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) val = (val & ~ZYNQMP_DMA_ARCACHE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_ARCACHE_OFST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) val = (val & ~ZYNQMP_DMA_AWCACHE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) (ZYNQMP_DMA_AXCACHE_VAL << ZYNQMP_DMA_AWCACHE_OFST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* Clearing the interrupt account rgisters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) chan->idle = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * zynqmp_dma_tx_submit - Submit DMA transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * @tx: Async transaction descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Return: cookie value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static dma_cookie_t zynqmp_dma_tx_submit(struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct zynqmp_dma_chan *chan = to_chan(tx->chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct zynqmp_dma_desc_sw *desc, *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) dma_cookie_t cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) new = tx_to_desc(tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) cookie = dma_cookie_assign(tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!list_empty(&chan->pending_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) desc = list_last_entry(&chan->pending_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) struct zynqmp_dma_desc_sw, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!list_empty(&desc->tx_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) desc = list_last_entry(&desc->tx_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct zynqmp_dma_desc_sw, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) desc->src_v->nxtdscraddr = new->src_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) desc->src_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) desc->dst_v->nxtdscraddr = new->dst_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) desc->dst_v->ctrl &= ~ZYNQMP_DMA_DESC_CTRL_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) list_add_tail(&new->node, &chan->pending_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * zynqmp_dma_get_descriptor - Get the sw descriptor from the pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Return: The sw descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static struct zynqmp_dma_desc_sw *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct zynqmp_dma_desc_sw *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) desc = list_first_entry(&chan->free_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct zynqmp_dma_desc_sw, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) list_del(&desc->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) INIT_LIST_HEAD(&desc->tx_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* Clear the src and dst descriptor memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) return desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * zynqmp_dma_free_descriptor - Issue pending transactions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * @sdesc: Transaction descriptor pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct zynqmp_dma_desc_sw *sdesc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct zynqmp_dma_desc_sw *child, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) chan->desc_free_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) list_del(&sdesc->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) list_add_tail(&sdesc->node, &chan->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) chan->desc_free_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) list_move_tail(&child->node, &chan->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * zynqmp_dma_free_desc_list - Free descriptors list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * @list: List to parse and delete the descriptor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct zynqmp_dma_desc_sw *desc, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) list_for_each_entry_safe(desc, next, list, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) zynqmp_dma_free_descriptor(chan, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * zynqmp_dma_alloc_chan_resources - Allocate channel resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * @dchan: DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * Return: Number of descriptors on success and failure value on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) struct zynqmp_dma_chan *chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct zynqmp_dma_desc_sw *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) ret = pm_runtime_resume_and_get(chan->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (!chan->sw_desc_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) chan->idle = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) INIT_LIST_HEAD(&chan->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) desc = chan->sw_desc_pool + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) desc->async_tx.tx_submit = zynqmp_dma_tx_submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) list_add_tail(&desc->node, &chan->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) chan->desc_pool_v = dma_alloc_coherent(chan->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) &chan->desc_pool_p, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!chan->desc_pool_v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) for (i = 0; i < ZYNQMP_DMA_NUM_DESCS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) desc = chan->sw_desc_pool + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) desc->dst_v = (struct zynqmp_dma_desc_ll *) (desc->src_v + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) desc->src_p = chan->desc_pool_p +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return ZYNQMP_DMA_NUM_DESCS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * zynqmp_dma_start - Start DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static void zynqmp_dma_start(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) chan->idle = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * zynqmp_dma_handle_ovfl_int - Process the overflow interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * @status: Interrupt status value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (status & ZYNQMP_DMA_BYTE_CNT_OVRFL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (status & ZYNQMP_DMA_IRQ_DST_ACCT_ERR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (status & ZYNQMP_DMA_IRQ_SRC_ACCT_ERR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) u32 val, burst_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) val = readl(chan->regs + ZYNQMP_DMA_CTRL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) val |= ZYNQMP_DMA_POINT_TYPE_SG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) writel(val, chan->regs + ZYNQMP_DMA_CTRL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) burst_val = __ilog2_u32(chan->src_burst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) val = (val & ~ZYNQMP_DMA_ARLEN) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) ((burst_val << ZYNQMP_DMA_ARLEN_OFST) & ZYNQMP_DMA_ARLEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) burst_val = __ilog2_u32(chan->dst_burst_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) val = (val & ~ZYNQMP_DMA_AWLEN) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) ((burst_val << ZYNQMP_DMA_AWLEN_OFST) & ZYNQMP_DMA_AWLEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * zynqmp_dma_device_config - Zynqmp dma device configuration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * @dchan: DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * @config: DMA device config
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * Return: 0 always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static int zynqmp_dma_device_config(struct dma_chan *dchan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct dma_slave_config *config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct zynqmp_dma_chan *chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) chan->src_burst_len = clamp(config->src_maxburst, 1U,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) ZYNQMP_DMA_MAX_SRC_BURST_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) chan->dst_burst_len = clamp(config->dst_maxburst, 1U,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) ZYNQMP_DMA_MAX_DST_BURST_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * zynqmp_dma_start_transfer - Initiate the new transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) struct zynqmp_dma_desc_sw *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (!chan->idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) zynqmp_dma_config(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) desc = list_first_entry_or_null(&chan->pending_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct zynqmp_dma_desc_sw, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (!desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) list_splice_tail_init(&chan->pending_list, &chan->active_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) zynqmp_dma_update_desc_to_ctrlr(chan, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) zynqmp_dma_start(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * zynqmp_dma_chan_desc_cleanup - Cleanup the completed descriptors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * @chan: ZynqMP DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct zynqmp_dma_desc_sw *desc, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) list_for_each_entry_safe(desc, next, &chan->done_list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) dma_async_tx_callback callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void *callback_param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) callback = desc->async_tx.callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) callback_param = desc->async_tx.callback_param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (callback) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) spin_unlock(&chan->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) callback(callback_param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) spin_lock(&chan->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /* Run any dependencies, then free the descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) zynqmp_dma_free_descriptor(chan, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * zynqmp_dma_complete_descriptor - Mark the active descriptor as complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct zynqmp_dma_desc_sw *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) desc = list_first_entry_or_null(&chan->active_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct zynqmp_dma_desc_sw, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (!desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) list_del(&desc->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) dma_cookie_complete(&desc->async_tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) list_add_tail(&desc->node, &chan->done_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * zynqmp_dma_issue_pending - Issue pending transactions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * @dchan: DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) static void zynqmp_dma_issue_pending(struct dma_chan *dchan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct zynqmp_dma_chan *chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) zynqmp_dma_start_transfer(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * zynqmp_dma_free_descriptors - Free channel descriptors
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) zynqmp_dma_free_desc_list(chan, &chan->active_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) zynqmp_dma_free_desc_list(chan, &chan->pending_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) zynqmp_dma_free_desc_list(chan, &chan->done_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * zynqmp_dma_free_chan_resources - Free channel resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * @dchan: DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct zynqmp_dma_chan *chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) zynqmp_dma_free_descriptors(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) dma_free_coherent(chan->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) chan->desc_pool_v, chan->desc_pool_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) kfree(chan->sw_desc_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) pm_runtime_mark_last_busy(chan->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) pm_runtime_put_autosuspend(chan->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) * zynqmp_dma_reset - Reset the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) zynqmp_dma_complete_descriptor(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) zynqmp_dma_chan_desc_cleanup(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) zynqmp_dma_free_descriptors(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) zynqmp_dma_init(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * zynqmp_dma_irq_handler - ZynqMP DMA Interrupt handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * @irq: IRQ number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * @data: Pointer to the ZynqMP DMA channel structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * Return: IRQ_HANDLED/IRQ_NONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) u32 isr, imr, status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) irqreturn_t ret = IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) isr = readl(chan->regs + ZYNQMP_DMA_ISR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) imr = readl(chan->regs + ZYNQMP_DMA_IMR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) status = isr & ~imr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) writel(isr, chan->regs + ZYNQMP_DMA_ISR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (status & ZYNQMP_DMA_INT_DONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) tasklet_schedule(&chan->tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) ret = IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (status & ZYNQMP_DMA_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) chan->idle = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (status & ZYNQMP_DMA_INT_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) chan->err = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) tasklet_schedule(&chan->tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) dev_err(chan->dev, "Channel %p has errors\n", chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) ret = IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (status & ZYNQMP_DMA_INT_OVRFL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) zynqmp_dma_handle_ovfl_int(chan, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) ret = IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * zynqmp_dma_do_tasklet - Schedule completion tasklet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * @t: Pointer to the ZynqMP DMA channel structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) static void zynqmp_dma_do_tasklet(struct tasklet_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) struct zynqmp_dma_chan *chan = from_tasklet(chan, t, tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) u32 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (chan->err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) zynqmp_dma_reset(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) chan->err = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) while (count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) zynqmp_dma_complete_descriptor(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) zynqmp_dma_chan_desc_cleanup(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (chan->idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) zynqmp_dma_start_transfer(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * zynqmp_dma_device_terminate_all - Aborts all transfers on a channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * @dchan: DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * Return: Always '0'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct zynqmp_dma_chan *chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) zynqmp_dma_free_descriptors(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * @dchan: DMA channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * @dma_dst: Destination buffer address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * @dma_src: Source buffer address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * @len: Transfer length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * @flags: transfer ack flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * Return: Async transaction descriptor on success and NULL on failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) struct dma_chan *dchan, dma_addr_t dma_dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) dma_addr_t dma_src, size_t len, ulong flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) struct zynqmp_dma_chan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) struct zynqmp_dma_desc_sw *new, *first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) void *desc = NULL, *prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) size_t copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) u32 desc_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) chan = to_chan(dchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) desc_cnt = DIV_ROUND_UP(len, ZYNQMP_DMA_MAX_TRANS_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) spin_lock_irqsave(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (desc_cnt > chan->desc_free_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) dev_dbg(chan->dev, "chan %p descs are not available\n", chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) spin_unlock_irqrestore(&chan->lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /* Allocate and populate the descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) new = zynqmp_dma_get_descriptor(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) copy = min_t(size_t, len, ZYNQMP_DMA_MAX_TRANS_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) desc = (struct zynqmp_dma_desc_ll *)new->src_v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) dma_dst, copy, prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) prev = desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) len -= copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) dma_src += copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) dma_dst += copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) first = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) list_add_tail(&new->node, &first->tx_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) } while (len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) zynqmp_dma_desc_config_eod(chan, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) async_tx_ack(&first->async_tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) first->async_tx.flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return &first->async_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * zynqmp_dma_chan_remove - Channel remove function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * @chan: ZynqMP DMA channel pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) if (chan->irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) devm_free_irq(chan->zdev->dev, chan->irq, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) tasklet_kill(&chan->tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) list_del(&chan->common.device_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * zynqmp_dma_chan_probe - Per Channel Probing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * @zdev: Driver specific device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * @pdev: Pointer to the platform_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * Return: '0' on success and failure value on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) struct zynqmp_dma_chan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct device_node *node = pdev->dev.of_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) if (!chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) chan->dev = zdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) chan->zdev = zdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) chan->regs = devm_ioremap_resource(&pdev->dev, res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (IS_ERR(chan->regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return PTR_ERR(chan->regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) dev_err(&pdev->dev, "missing xlnx,bus-width property\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) dev_err(zdev->dev, "invalid bus-width value");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) zdev->chan = chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) spin_lock_init(&chan->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) INIT_LIST_HEAD(&chan->active_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) INIT_LIST_HEAD(&chan->pending_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) INIT_LIST_HEAD(&chan->done_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) INIT_LIST_HEAD(&chan->free_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) dma_cookie_init(&chan->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) chan->common.device = &zdev->common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) list_add_tail(&chan->common.device_node, &zdev->common.channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) zynqmp_dma_init(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) chan->irq = platform_get_irq(pdev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (chan->irq < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) "zynqmp-dma", chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) chan->idle = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * of_zynqmp_dma_xlate - Translation function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * @dma_spec: Pointer to DMA specifier as found in the device tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * @ofdma: Pointer to DMA controller data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * Return: DMA channel pointer on success and NULL on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) struct of_dma *ofdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) struct zynqmp_dma_device *zdev = ofdma->of_dma_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) return dma_get_slave_channel(&zdev->chan->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * zynqmp_dma_suspend - Suspend method for the driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * @dev: Address of the device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * Put the driver into low power mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * Return: 0 on success and failure value on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) static int __maybe_unused zynqmp_dma_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) if (!device_may_wakeup(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) return pm_runtime_force_suspend(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * zynqmp_dma_resume - Resume from suspend
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) * @dev: Address of the device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * Resume operation after suspend.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * Return: 0 on success and failure value on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) static int __maybe_unused zynqmp_dma_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) if (!device_may_wakeup(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) return pm_runtime_force_resume(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * @dev: Address of the device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * Put the driver into low power mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) * Return: 0 always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) clk_disable_unprepare(zdev->clk_main);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) clk_disable_unprepare(zdev->clk_apb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * @dev: Address of the device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) * Put the driver into low power mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * Return: 0 always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) err = clk_prepare_enable(zdev->clk_main);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) dev_err(dev, "Unable to enable main clock.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) err = clk_prepare_enable(zdev->clk_apb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) dev_err(dev, "Unable to enable apb clock.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) clk_disable_unprepare(zdev->clk_main);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) static const struct dev_pm_ops zynqmp_dma_dev_pm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) SET_SYSTEM_SLEEP_PM_OPS(zynqmp_dma_suspend, zynqmp_dma_resume)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) SET_RUNTIME_PM_OPS(zynqmp_dma_runtime_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) zynqmp_dma_runtime_resume, NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * zynqmp_dma_probe - Driver probe function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * @pdev: Pointer to the platform_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * Return: '0' on success and failure value on error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) static int zynqmp_dma_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct zynqmp_dma_device *zdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) struct dma_device *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) zdev = devm_kzalloc(&pdev->dev, sizeof(*zdev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (!zdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) zdev->dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) INIT_LIST_HEAD(&zdev->common.channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) p = &zdev->common;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) p->device_terminate_all = zynqmp_dma_device_terminate_all;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) p->device_issue_pending = zynqmp_dma_issue_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) p->device_free_chan_resources = zynqmp_dma_free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) p->device_tx_status = dma_cookie_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) p->device_config = zynqmp_dma_device_config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) p->dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (IS_ERR(zdev->clk_main)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) dev_err(&pdev->dev, "main clock not found.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return PTR_ERR(zdev->clk_main);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (IS_ERR(zdev->clk_apb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) dev_err(&pdev->dev, "apb clock not found.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) return PTR_ERR(zdev->clk_apb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) platform_set_drvdata(pdev, zdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) pm_runtime_use_autosuspend(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) pm_runtime_enable(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) pm_runtime_get_sync(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) if (!pm_runtime_enabled(zdev->dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) ret = zynqmp_dma_runtime_resume(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) ret = zynqmp_dma_chan_probe(zdev, pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) dev_err(&pdev->dev, "Probing channel failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) goto err_disable_pm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) p->src_addr_widths = BIT(zdev->chan->bus_width / 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) dma_async_device_register(&zdev->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) ret = of_dma_controller_register(pdev->dev.of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) of_zynqmp_dma_xlate, zdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) dev_err(&pdev->dev, "Unable to register DMA to DT\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) dma_async_device_unregister(&zdev->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) goto free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) pm_runtime_mark_last_busy(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) pm_runtime_put_sync_autosuspend(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) free_chan_resources:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) zynqmp_dma_chan_remove(zdev->chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) err_disable_pm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (!pm_runtime_enabled(zdev->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) zynqmp_dma_runtime_suspend(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) pm_runtime_disable(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) * zynqmp_dma_remove - Driver remove function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * @pdev: Pointer to the platform_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) * Return: Always '0'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) static int zynqmp_dma_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) of_dma_controller_free(pdev->dev.of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) dma_async_device_unregister(&zdev->common);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) zynqmp_dma_chan_remove(zdev->chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) pm_runtime_disable(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (!pm_runtime_enabled(zdev->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) zynqmp_dma_runtime_suspend(zdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) static const struct of_device_id zynqmp_dma_of_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) { .compatible = "xlnx,zynqmp-dma-1.0", },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) MODULE_DEVICE_TABLE(of, zynqmp_dma_of_match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) static struct platform_driver zynqmp_dma_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) .name = "xilinx-zynqmp-dma",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) .of_match_table = zynqmp_dma_of_match,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) .pm = &zynqmp_dma_dev_pm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) .probe = zynqmp_dma_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) .remove = zynqmp_dma_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) module_platform_driver(zynqmp_dma_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) MODULE_AUTHOR("Xilinx, Inc.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) MODULE_DESCRIPTION("Xilinx ZynqMP DMA driver");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags