^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/of_dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/of_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/dmapool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/of_address.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "../dmaengine.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define DESC_TYPE 27
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define DESC_TYPE_HOST 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define DESC_TYPE_TEARD 0x13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define TD_DESC_IS_RX (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define TD_DESC_DMA_NUM 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define DESC_LENGTH_BITS_NUM 21
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define DESC_TYPE_USB (5 << 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define DESC_PD_COMPLETE (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* DMA engine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define DMA_TDFDQ 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define DMA_TXGCR(x) (0x800 + (x) * 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define DMA_RXGCR(x) (0x808 + (x) * 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define RXHPCRA0 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define GCR_CHAN_ENABLE (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define GCR_TEARDOWN (1 << 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define GCR_STARV_RETRY (1 << 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define GCR_DESC_TYPE_HOST (1 << 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /* DMA scheduler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define DMA_SCHED_CTRL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define DMA_SCHED_CTRL_EN (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define DMA_SCHED_WORD(x) ((x) * 4 + 0x800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define SCHED_ENTRY0_CHAN(x) ((x) << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define SCHED_ENTRY0_IS_RX (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define SCHED_ENTRY1_CHAN(x) ((x) << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define SCHED_ENTRY1_IS_RX (1 << 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define SCHED_ENTRY2_CHAN(x) ((x) << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define SCHED_ENTRY2_IS_RX (1 << 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define SCHED_ENTRY3_CHAN(x) ((x) << 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define SCHED_ENTRY3_IS_RX (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /* Queue manager */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* 4 KiB of memory for descriptors, 2 for each endpoint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define ALLOC_DECS_NUM 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define DESCS_AREAS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define TOTAL_DESCS_NUM (ALLOC_DECS_NUM * DESCS_AREAS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define QMGR_SCRATCH_SIZE (TOTAL_DESCS_NUM * 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define QMGR_LRAM0_BASE 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define QMGR_LRAM_SIZE 0x84
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define QMGR_LRAM1_BASE 0x88
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define QMGR_MEMBASE(x) (0x1000 + (x) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define QMGR_MEMCTRL(x) (0x1004 + (x) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define QMGR_MEMCTRL_IDX_SH 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define QMGR_MEMCTRL_DESC_SH 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define QMGR_PEND(x) (0x90 + (x) * 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define QMGR_PENDING_SLOT_Q(x) (x / 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define QMGR_PENDING_BIT_Q(x) (x % 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /* Packet Descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define PD2_ZERO_LENGTH (1 << 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct cppi41_channel {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) struct dma_chan chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct dma_async_tx_descriptor txd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct cppi41_dd *cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct cppi41_desc *desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) dma_addr_t desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) void __iomem *gcr_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) int is_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) u32 residue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) unsigned int q_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) unsigned int q_comp_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) unsigned int port_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned td_retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned td_queued:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) unsigned td_seen:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned td_desc_seen:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct list_head node; /* Node for pending list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct cppi41_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) u32 pd0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) u32 pd1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) u32 pd2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) u32 pd3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) u32 pd4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) u32 pd5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) u32 pd6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) u32 pd7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) } __aligned(32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct chan_queues {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) u16 submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) u16 complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct cppi41_dd {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct dma_device ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) void *qmgr_scratch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) dma_addr_t scratch_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct cppi41_desc *cd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) dma_addr_t descs_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) u32 first_td_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) void __iomem *ctrl_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) void __iomem *sched_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) void __iomem *qmgr_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unsigned int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) const struct chan_queues *queues_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) const struct chan_queues *queues_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct chan_queues td_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) u16 first_completion_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) u16 qmgr_num_pend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) u32 n_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u8 platform;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct list_head pending; /* Pending queued transfers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) spinlock_t lock; /* Lock for pending list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* context for suspend/resume */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unsigned int dma_tdfdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) bool is_suspended;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static struct chan_queues am335x_usb_queues_tx[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /* USB0 ENDP 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) [ 0] = { .submit = 32, .complete = 93},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) [ 1] = { .submit = 34, .complete = 94},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) [ 2] = { .submit = 36, .complete = 95},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) [ 3] = { .submit = 38, .complete = 96},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) [ 4] = { .submit = 40, .complete = 97},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) [ 5] = { .submit = 42, .complete = 98},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) [ 6] = { .submit = 44, .complete = 99},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) [ 7] = { .submit = 46, .complete = 100},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) [ 8] = { .submit = 48, .complete = 101},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) [ 9] = { .submit = 50, .complete = 102},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) [10] = { .submit = 52, .complete = 103},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) [11] = { .submit = 54, .complete = 104},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) [12] = { .submit = 56, .complete = 105},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) [13] = { .submit = 58, .complete = 106},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) [14] = { .submit = 60, .complete = 107},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* USB1 ENDP1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) [15] = { .submit = 62, .complete = 125},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) [16] = { .submit = 64, .complete = 126},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) [17] = { .submit = 66, .complete = 127},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) [18] = { .submit = 68, .complete = 128},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) [19] = { .submit = 70, .complete = 129},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) [20] = { .submit = 72, .complete = 130},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) [21] = { .submit = 74, .complete = 131},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) [22] = { .submit = 76, .complete = 132},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) [23] = { .submit = 78, .complete = 133},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) [24] = { .submit = 80, .complete = 134},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) [25] = { .submit = 82, .complete = 135},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) [26] = { .submit = 84, .complete = 136},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) [27] = { .submit = 86, .complete = 137},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) [28] = { .submit = 88, .complete = 138},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) [29] = { .submit = 90, .complete = 139},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static const struct chan_queues am335x_usb_queues_rx[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* USB0 ENDP 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) [ 0] = { .submit = 1, .complete = 109},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) [ 1] = { .submit = 2, .complete = 110},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) [ 2] = { .submit = 3, .complete = 111},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) [ 3] = { .submit = 4, .complete = 112},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) [ 4] = { .submit = 5, .complete = 113},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) [ 5] = { .submit = 6, .complete = 114},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) [ 6] = { .submit = 7, .complete = 115},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) [ 7] = { .submit = 8, .complete = 116},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) [ 8] = { .submit = 9, .complete = 117},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) [ 9] = { .submit = 10, .complete = 118},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) [10] = { .submit = 11, .complete = 119},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) [11] = { .submit = 12, .complete = 120},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) [12] = { .submit = 13, .complete = 121},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) [13] = { .submit = 14, .complete = 122},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) [14] = { .submit = 15, .complete = 123},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /* USB1 ENDP 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) [15] = { .submit = 16, .complete = 141},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) [16] = { .submit = 17, .complete = 142},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) [17] = { .submit = 18, .complete = 143},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) [18] = { .submit = 19, .complete = 144},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) [19] = { .submit = 20, .complete = 145},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) [20] = { .submit = 21, .complete = 146},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) [21] = { .submit = 22, .complete = 147},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) [22] = { .submit = 23, .complete = 148},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) [23] = { .submit = 24, .complete = 149},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) [24] = { .submit = 25, .complete = 150},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) [25] = { .submit = 26, .complete = 151},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) [26] = { .submit = 27, .complete = 152},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) [27] = { .submit = 28, .complete = 153},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) [28] = { .submit = 29, .complete = 154},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) [29] = { .submit = 30, .complete = 155},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static const struct chan_queues da8xx_usb_queues_tx[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) [0] = { .submit = 16, .complete = 24},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) [1] = { .submit = 18, .complete = 24},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) [2] = { .submit = 20, .complete = 24},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) [3] = { .submit = 22, .complete = 24},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static const struct chan_queues da8xx_usb_queues_rx[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) [0] = { .submit = 1, .complete = 26},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) [1] = { .submit = 3, .complete = 26},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) [2] = { .submit = 5, .complete = 26},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) [3] = { .submit = 7, .complete = 26},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) struct cppi_glue_infos {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) const struct chan_queues *queues_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) const struct chan_queues *queues_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) struct chan_queues td_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) u16 first_completion_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) u16 qmgr_num_pend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return container_of(c, struct cppi41_channel, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct cppi41_channel *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) u32 descs_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) u32 desc_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (!((desc >= cdd->descs_phys) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) (desc < (cdd->descs_phys + descs_size)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) BUG_ON(desc_num >= ALLOC_DECS_NUM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) c = cdd->chan_busy[desc_num];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) cdd->chan_busy[desc_num] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* Usecount for chan_busy[], paired with push_desc_queue() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pm_runtime_put(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static void cppi_writel(u32 val, void *__iomem *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) __raw_writel(val, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static u32 cppi_readl(void *__iomem *mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return __raw_readl(mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static u32 pd_trans_len(u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) u32 desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) desc &= ~0x1f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static irqreturn_t cppi41_irq(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) struct cppi41_dd *cdd = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) u16 first_completion_queue = cdd->first_completion_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) u16 qmgr_num_pend = cdd->qmgr_num_pend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct cppi41_channel *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) for (i = QMGR_PENDING_SLOT_Q(first_completion_queue); i < qmgr_num_pend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) u32 q_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (i == QMGR_PENDING_SLOT_Q(first_completion_queue) && val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) /* set corresponding bit for completetion Q 93 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) mask = 1 << QMGR_PENDING_BIT_Q(first_completion_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* not set all bits for queues less than Q 93 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) mask--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* now invert and keep only Q 93+ set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) val &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) __iormb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) while (val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) u32 desc, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * This should never trigger, see the comments in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * push_desc_queue()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) WARN_ON(cdd->is_suspended);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) q_num = __fls(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) val &= ~(1 << q_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) q_num += 32 * i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) desc = cppi41_pop_desc(cdd, q_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) c = desc_to_chan(cdd, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (WARN_ON(!c)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) pr_err("%s() q %d desc %08x\n", __func__,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) q_num, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (c->desc->pd2 & PD2_ZERO_LENGTH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) len = pd_trans_len(c->desc->pd0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) c->residue = pd_trans_len(c->desc->pd6) - len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) dma_cookie_complete(&c->txd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) dmaengine_desc_get_callback_invoke(&c->txd, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) dma_cookie_t cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) cookie = dma_cookie_assign(tx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) error = pm_runtime_get_sync(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (error < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) __func__, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) pm_runtime_put_noidle(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) dma_cookie_init(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) dma_async_tx_descriptor_init(&c->txd, chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) c->txd.tx_submit = cppi41_tx_submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (!c->is_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) pm_runtime_mark_last_busy(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) pm_runtime_put_autosuspend(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) error = pm_runtime_get_sync(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (error < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) pm_runtime_put_noidle(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) WARN_ON(!list_empty(&cdd->pending));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) pm_runtime_mark_last_busy(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) pm_runtime_put_autosuspend(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) dma_cookie_t cookie, struct dma_tx_state *txstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) enum dma_status ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) ret = dma_cookie_status(chan, cookie, txstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) dma_set_residue(txstate, c->residue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static void push_desc_queue(struct cppi41_channel *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) u32 desc_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) u32 desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) c->residue = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) reg = GCR_CHAN_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (!c->is_tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) reg |= GCR_STARV_RETRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) reg |= GCR_DESC_TYPE_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) reg |= c->q_comp_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) cppi_writel(reg, c->gcr_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * We don't use writel() but __raw_writel() so we have to make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * that the DMA descriptor in coherent memory made to the main memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * before starting the dma engine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) __iowmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * DMA transfers can take at least 200ms to complete with USB mass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * storage connected. To prevent autosuspend timeouts, we must use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * pm_runtime_get/put() when chan_busy[] is modified. This will get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * cleared in desc_to_chan() or cppi41_stop_chan() depending on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * outcome of the transfer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) pm_runtime_get(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) desc_phys = lower_32_bits(c->desc_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) WARN_ON(cdd->chan_busy[desc_num]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) cdd->chan_busy[desc_num] = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) reg = (sizeof(struct cppi41_desc) - 24) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) reg |= desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * Caller must hold cdd->lock to prevent push_desc_queue()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * getting called out of order. We have both cppi41_dma_issue_pending()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * and cppi41_runtime_resume() call this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) static void cppi41_run_queue(struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct cppi41_channel *c, *_c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) list_for_each_entry_safe(c, _c, &cdd->pending, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) push_desc_queue(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) list_del(&c->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) static void cppi41_dma_issue_pending(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) error = pm_runtime_get(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if ((error != -EINPROGRESS) && error < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) pm_runtime_put_noidle(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) spin_lock_irqsave(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) list_add_tail(&c->node, &cdd->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (!cdd->is_suspended)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) cppi41_run_queue(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) spin_unlock_irqrestore(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) pm_runtime_mark_last_busy(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) pm_runtime_put_autosuspend(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static u32 get_host_pd0(u32 length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) reg = DESC_TYPE_HOST << DESC_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) reg |= length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static u32 get_host_pd1(struct cppi41_channel *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) reg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static u32 get_host_pd2(struct cppi41_channel *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) reg = DESC_TYPE_USB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) reg |= c->q_comp_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static u32 get_host_pd3(u32 length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /* PD3 = packet size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) reg = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) static u32 get_host_pd6(u32 length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* PD6 buffer size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) reg = DESC_PD_COMPLETE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) reg |= length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) static u32 get_host_pd4_or_7(u32 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) reg = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static u32 get_host_pd5(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) reg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct dma_async_tx_descriptor *txd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct cppi41_desc *d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) error = pm_runtime_get(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (error < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) pm_runtime_put_noidle(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (cdd->is_suspended)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) goto err_out_not_ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) d = c->desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) for_each_sg(sgl, sg, sg_len, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u32 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) u32 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* We need to use more than one desc once musb supports sg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) addr = lower_32_bits(sg_dma_address(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) len = sg_dma_len(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) d->pd0 = get_host_pd0(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) d->pd1 = get_host_pd1(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) d->pd2 = get_host_pd2(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) d->pd3 = get_host_pd3(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) d->pd4 = get_host_pd4_or_7(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) d->pd5 = get_host_pd5();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) d->pd6 = get_host_pd6(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) d->pd7 = get_host_pd4_or_7(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) d++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) txd = &c->txd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) err_out_not_ready:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) pm_runtime_mark_last_busy(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) pm_runtime_put_autosuspend(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) return txd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) static void cppi41_compute_td_desc(struct cppi41_desc *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) static int cppi41_tear_down_chan(struct cppi41_channel *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct dmaengine_result abort_result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct cppi41_desc *td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) u32 desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) u32 td_desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) td = cdd->cd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) td += cdd->first_td_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) td_desc_phys = cdd->descs_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (!c->td_queued) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) cppi41_compute_td_desc(td);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) __iowmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) reg = (sizeof(struct cppi41_desc) - 24) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) reg |= td_desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) cppi_writel(reg, cdd->qmgr_mem +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) QMGR_QUEUE_D(cdd->td_queue.submit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) reg = GCR_CHAN_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (!c->is_tx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) reg |= GCR_STARV_RETRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) reg |= GCR_DESC_TYPE_HOST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) reg |= cdd->td_queue.complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) reg |= GCR_TEARDOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) cppi_writel(reg, c->gcr_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) c->td_queued = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) c->td_retry = 500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (!c->td_seen || !c->td_desc_seen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (!desc_phys && c->is_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (desc_phys == c->desc_phys) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) c->td_desc_seen = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) } else if (desc_phys == td_desc_phys) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) u32 pd0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) __iormb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) pd0 = td->pd0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) WARN_ON((pd0 & 0x1f) != c->port_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) c->td_seen = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) } else if (desc_phys) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) c->td_retry--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * If the TX descriptor / channel is in use, the caller needs to poke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * his TD bit multiple times. After that he hardware releases the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * transfer descriptor followed by TD descriptor. Waiting seems not to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * cause any difference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * RX seems to be thrown out right away. However once the TearDown
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) * descriptor gets through we are done. If we have seens the transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * descriptor before the TD we fetch it from enqueue, it has to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * there waiting for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (!c->td_seen && c->td_retry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) WARN_ON(!c->td_retry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (!c->td_desc_seen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) desc_phys = cppi41_pop_desc(cdd, c->q_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (!desc_phys)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) WARN_ON(!desc_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) c->td_queued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) c->td_seen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) c->td_desc_seen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) cppi_writel(0, c->gcr_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) /* Invoke the callback to do the necessary clean-up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) abort_result.result = DMA_TRANS_ABORTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) dma_cookie_complete(&c->txd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) dmaengine_desc_get_callback_invoke(&c->txd, &abort_result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) static int cppi41_stop_chan(struct dma_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) struct cppi41_channel *c = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) struct cppi41_dd *cdd = c->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) u32 desc_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) u32 desc_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) desc_phys = lower_32_bits(c->desc_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (!cdd->chan_busy[desc_num]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) struct cppi41_channel *cc, *_ct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * channels might still be in the pendling list if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * cppi41_dma_issue_pending() is called after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * cppi41_runtime_suspend() is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) list_for_each_entry_safe(cc, _ct, &cdd->pending, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (cc != c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) list_del(&cc->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) ret = cppi41_tear_down_chan(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) WARN_ON(!cdd->chan_busy[desc_num]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) cdd->chan_busy[desc_num] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) /* Usecount for chan_busy[], paired with push_desc_queue() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) pm_runtime_put(cdd->ddev.dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) struct cppi41_channel *cchan, *chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) u32 n_chans = cdd->n_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * The channels can only be used as TX or as RX. So we add twice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * that much dma channels because USB can only do RX or TX.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) n_chans *= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) chans = devm_kcalloc(dev, n_chans, sizeof(*chans), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (!chans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) for (i = 0; i < n_chans; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) cchan = &chans[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) cchan->cdd = cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (i & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) cchan->is_tx = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) cchan->is_tx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) cchan->port_num = i >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) cchan->desc = &cdd->cd[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) cchan->desc_phys = cdd->descs_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) cchan->desc_phys += i * sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) cchan->chan.device = &cdd->ddev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) cdd->first_td_desc = n_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) static void purge_descs(struct device *dev, struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) unsigned int mem_decs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) for (i = 0; i < DESCS_AREAS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) dma_free_coherent(dev, mem_decs, cdd->cd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) cdd->descs_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) static void disable_sched(struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) disable_sched(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) purge_descs(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) cdd->scratch_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) static int init_descs(struct device *dev, struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) unsigned int desc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) unsigned int mem_decs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) BUILD_BUG_ON(sizeof(struct cppi41_desc) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) (sizeof(struct cppi41_desc) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) desc_size = sizeof(struct cppi41_desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) mem_decs = ALLOC_DECS_NUM * desc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) for (i = 0; i < DESCS_AREAS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) reg = idx << QMGR_MEMCTRL_IDX_SH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) reg |= ilog2(ALLOC_DECS_NUM) - 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) BUILD_BUG_ON(DESCS_AREAS != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) cdd->cd = dma_alloc_coherent(dev, mem_decs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) &cdd->descs_phys, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (!cdd->cd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) idx += ALLOC_DECS_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) static void init_sched(struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) unsigned ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) unsigned word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) word = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) for (ch = 0; ch < cdd->n_chans; ch += 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) reg = SCHED_ENTRY0_CHAN(ch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) reg |= SCHED_ENTRY2_CHAN(ch + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) word++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) reg = cdd->n_chans * 2 - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) reg |= DMA_SCHED_CTRL_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) &cdd->scratch_phys, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (!cdd->qmgr_scratch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) ret = init_descs(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) goto err_td;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) init_sched(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) err_td:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) deinit_cppi41(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) static struct platform_driver cpp41_dma_driver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * The param format is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * X Y
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * X: Port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * Y: 0 = RX else TX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) #define INFO_PORT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) #define INFO_IS_TX 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) struct cppi41_channel *cchan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) struct cppi41_dd *cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) const struct chan_queues *queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) u32 *num = param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) if (chan->device->dev->driver != &cpp41_dma_driver.driver)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) cchan = to_cpp41_chan(chan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) if (cchan->port_num != num[INFO_PORT])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (cchan->is_tx && !num[INFO_IS_TX])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) cdd = cchan->cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) if (cchan->is_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) queues = cdd->queues_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) queues = cdd->queues_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) BUILD_BUG_ON(ARRAY_SIZE(am335x_usb_queues_rx) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) ARRAY_SIZE(am335x_usb_queues_tx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) if (WARN_ON(cchan->port_num >= ARRAY_SIZE(am335x_usb_queues_rx)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) cchan->q_num = queues[cchan->port_num].submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) cchan->q_comp_num = queues[cchan->port_num].complete;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) static struct of_dma_filter_info cpp41_dma_info = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) .filter_fn = cpp41_dma_filter_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct of_dma *ofdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) int count = dma_spec->args_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct of_dma_filter_info *info = ofdma->of_dma_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) if (!info || !info->filter_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (count != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return dma_request_channel(info->dma_cap, info->filter_fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) &dma_spec->args[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) static const struct cppi_glue_infos am335x_usb_infos = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) .queues_rx = am335x_usb_queues_rx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) .queues_tx = am335x_usb_queues_tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) .td_queue = { .submit = 31, .complete = 0 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) .first_completion_queue = 93,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) .qmgr_num_pend = 5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) static const struct cppi_glue_infos da8xx_usb_infos = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) .queues_rx = da8xx_usb_queues_rx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) .queues_tx = da8xx_usb_queues_tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) .td_queue = { .submit = 31, .complete = 0 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) .first_completion_queue = 24,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) .qmgr_num_pend = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) static const struct of_device_id cppi41_dma_ids[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) { .compatible = "ti,am3359-cppi41", .data = &am335x_usb_infos},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) { .compatible = "ti,da830-cppi41", .data = &da8xx_usb_infos},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) static const struct cppi_glue_infos *get_glue_info(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) const struct of_device_id *of_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) of_id = of_match_node(cppi41_dma_ids, dev->of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (!of_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) return of_id->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) #define CPPI41_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) static int cppi41_dma_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) struct cppi41_dd *cdd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) const struct cppi_glue_infos *glue_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) struct resource *mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) glue_info = get_glue_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (!glue_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (!cdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) cdd->ddev.device_tx_status = cppi41_dma_tx_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) cdd->ddev.device_terminate_all = cppi41_stop_chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) cdd->ddev.dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) INIT_LIST_HEAD(&cdd->ddev.channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) index = of_property_match_string(dev->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) "reg-names", "controller");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (index < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) return index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) mem = platform_get_resource(pdev, IORESOURCE_MEM, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) cdd->ctrl_mem = devm_ioremap_resource(dev, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (IS_ERR(cdd->ctrl_mem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return PTR_ERR(cdd->ctrl_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) cdd->sched_mem = devm_ioremap_resource(dev, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (IS_ERR(cdd->sched_mem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) return PTR_ERR(cdd->sched_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) cdd->qmgr_mem = devm_ioremap_resource(dev, mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (IS_ERR(cdd->qmgr_mem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) return PTR_ERR(cdd->qmgr_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) spin_lock_init(&cdd->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) INIT_LIST_HEAD(&cdd->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) platform_set_drvdata(pdev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) pm_runtime_enable(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) pm_runtime_set_autosuspend_delay(dev, 100);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) pm_runtime_use_autosuspend(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) ret = pm_runtime_get_sync(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) goto err_get_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) cdd->queues_rx = glue_info->queues_rx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) cdd->queues_tx = glue_info->queues_tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) cdd->td_queue = glue_info->td_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) cdd->qmgr_num_pend = glue_info->qmgr_num_pend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) cdd->first_completion_queue = glue_info->first_completion_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) ret = of_property_read_u32(dev->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) "#dma-channels", &cdd->n_chans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) goto err_get_n_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) ret = init_cppi41(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) goto err_init_cppi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) ret = cppi41_add_chans(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) goto err_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) irq = irq_of_parse_and_map(dev->of_node, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) if (!irq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) goto err_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ret = devm_request_irq(&pdev->dev, irq, cppi41_irq, IRQF_SHARED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) dev_name(dev), cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) goto err_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) cdd->irq = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) ret = dma_async_device_register(&cdd->ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) goto err_chans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) ret = of_dma_controller_register(dev->of_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) cppi41_dma_xlate, &cpp41_dma_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) goto err_of;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) pm_runtime_mark_last_busy(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) pm_runtime_put_autosuspend(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) err_of:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) dma_async_device_unregister(&cdd->ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) err_chans:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) deinit_cppi41(dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) err_init_cppi:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) pm_runtime_dont_use_autosuspend(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) err_get_n_chans:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) err_get_sync:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) pm_runtime_put_sync(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) pm_runtime_disable(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) static int cppi41_dma_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct cppi41_dd *cdd = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) error = pm_runtime_get_sync(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) if (error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) dev_err(&pdev->dev, "%s could not pm_runtime_get: %i\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) __func__, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) of_dma_controller_free(pdev->dev.of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) dma_async_device_unregister(&cdd->ddev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) devm_free_irq(&pdev->dev, cdd->irq, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) deinit_cppi41(&pdev->dev, cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) pm_runtime_dont_use_autosuspend(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) pm_runtime_put_sync(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) pm_runtime_disable(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) static int __maybe_unused cppi41_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) struct cppi41_dd *cdd = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) disable_sched(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) static int __maybe_unused cppi41_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) struct cppi41_dd *cdd = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct cppi41_channel *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) for (i = 0; i < DESCS_AREAS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) if (!c->is_tx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) init_sched(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) static int __maybe_unused cppi41_runtime_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) struct cppi41_dd *cdd = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) spin_lock_irqsave(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) cdd->is_suspended = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) WARN_ON(!list_empty(&cdd->pending));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) spin_unlock_irqrestore(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) static int __maybe_unused cppi41_runtime_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) struct cppi41_dd *cdd = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) spin_lock_irqsave(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) cdd->is_suspended = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) cppi41_run_queue(cdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) spin_unlock_irqrestore(&cdd->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static const struct dev_pm_ops cppi41_pm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) SET_LATE_SYSTEM_SLEEP_PM_OPS(cppi41_suspend, cppi41_resume)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) SET_RUNTIME_PM_OPS(cppi41_runtime_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) cppi41_runtime_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) static struct platform_driver cpp41_dma_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) .probe = cppi41_dma_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) .remove = cppi41_dma_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) .name = "cppi41-dma-engine",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) .pm = &cppi41_pm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) .of_match_table = of_match_ptr(cppi41_dma_ids),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) module_platform_driver(cpp41_dma_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags