^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) // Copyright (c) 2018 MediaTek Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/clk-provider.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/iopoll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/mailbox_controller.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/mailbox/mtk-cmdq-mailbox.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/of_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define CMDQ_OP_CODE_MASK (0xff << CMDQ_OP_CODE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define CMDQ_NUM_CMD(t) (t->cmd_buf_size / CMDQ_INST_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define CMDQ_CURR_IRQ_STATUS 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define CMDQ_SYNC_TOKEN_UPDATE 0x68
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define CMDQ_THR_SLOT_CYCLES 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define CMDQ_THR_BASE 0x100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define CMDQ_THR_SIZE 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define CMDQ_THR_WARM_RESET 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define CMDQ_THR_ENABLE_TASK 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define CMDQ_THR_SUSPEND_TASK 0x08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define CMDQ_THR_CURR_STATUS 0x0c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define CMDQ_THR_IRQ_STATUS 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define CMDQ_THR_IRQ_ENABLE 0x14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define CMDQ_THR_CURR_ADDR 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define CMDQ_THR_END_ADDR 0x24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define CMDQ_THR_WAIT_TOKEN 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define CMDQ_THR_PRIORITY 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define CMDQ_THR_ACTIVE_SLOT_CYCLES 0x3200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define CMDQ_THR_ENABLED 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define CMDQ_THR_DISABLED 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define CMDQ_THR_SUSPEND 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define CMDQ_THR_RESUME 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define CMDQ_THR_STATUS_SUSPENDED BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define CMDQ_THR_DO_WARM_RESET BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define CMDQ_THR_IRQ_DONE 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define CMDQ_THR_IRQ_ERROR 0x12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define CMDQ_THR_IRQ_EN (CMDQ_THR_IRQ_ERROR | CMDQ_THR_IRQ_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define CMDQ_THR_IS_WAITING BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define CMDQ_JUMP_BY_OFFSET 0x10000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define CMDQ_JUMP_BY_PA 0x10000001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct cmdq_thread {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct mbox_chan *chan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct list_head task_busy_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) u32 priority;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct cmdq_task {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct cmdq *cmdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct list_head list_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) dma_addr_t pa_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct cmdq_thread *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct cmdq_pkt *pkt; /* the packet sent from mailbox client */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct cmdq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct mbox_controller mbox;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u32 thread_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) u32 irq_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct cmdq_thread *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct clk *clock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) bool suspended;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) u8 shift_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct gce_plat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) u32 thread_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) u8 shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u8 cmdq_get_shift_pa(struct mbox_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return cmdq->shift_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) EXPORT_SYMBOL(cmdq_get_shift_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int cmdq_thread_suspend(struct cmdq *cmdq, struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) writel(CMDQ_THR_SUSPEND, thread->base + CMDQ_THR_SUSPEND_TASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* If already disabled, treat as suspended successful. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_CURR_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) status, status & CMDQ_THR_STATUS_SUSPENDED, 0, 10)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) dev_err(cmdq->mbox.dev, "suspend GCE thread 0x%x failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) (u32)(thread->base - cmdq->base));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static void cmdq_thread_resume(struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) writel(CMDQ_THR_RESUME, thread->base + CMDQ_THR_SUSPEND_TASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static void cmdq_init(struct cmdq *cmdq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) WARN_ON(clk_enable(cmdq->clock) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) writel(CMDQ_THR_ACTIVE_SLOT_CYCLES, cmdq->base + CMDQ_THR_SLOT_CYCLES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) for (i = 0; i <= CMDQ_MAX_EVENT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) writel(i, cmdq->base + CMDQ_SYNC_TOKEN_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) clk_disable(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static int cmdq_thread_reset(struct cmdq *cmdq, struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) u32 warm_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) writel(CMDQ_THR_DO_WARM_RESET, thread->base + CMDQ_THR_WARM_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_WARM_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) warm_reset, !(warm_reset & CMDQ_THR_DO_WARM_RESET),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 0, 10)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) dev_err(cmdq->mbox.dev, "reset GCE thread 0x%x failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) (u32)(thread->base - cmdq->base));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void cmdq_thread_disable(struct cmdq *cmdq, struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) cmdq_thread_reset(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) writel(CMDQ_THR_DISABLED, thread->base + CMDQ_THR_ENABLE_TASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* notify GCE to re-fetch commands by setting GCE thread PC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static void cmdq_thread_invalidate_fetched_data(struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) writel(readl(thread->base + CMDQ_THR_CURR_ADDR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) thread->base + CMDQ_THR_CURR_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static void cmdq_task_insert_into_thread(struct cmdq_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct device *dev = task->cmdq->mbox.dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct cmdq_thread *thread = task->thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct cmdq_task *prev_task = list_last_entry(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) &thread->task_busy_list, typeof(*task), list_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) u64 *prev_task_base = prev_task->pkt->va_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /* let previous task jump to this task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) dma_sync_single_for_cpu(dev, prev_task->pa_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) (u64)CMDQ_JUMP_BY_PA << 32 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) (task->pa_base >> task->cmdq->shift_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) dma_sync_single_for_device(dev, prev_task->pa_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) cmdq_thread_invalidate_fetched_data(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static bool cmdq_thread_is_in_wfe(struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return readl(thread->base + CMDQ_THR_WAIT_TOKEN) & CMDQ_THR_IS_WAITING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static void cmdq_task_exec_done(struct cmdq_task *task, enum cmdq_cb_status sta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct cmdq_task_cb *cb = &task->pkt->async_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct cmdq_cb_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) WARN_ON(cb->cb == (cmdq_async_flush_cb)NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) data.sta = sta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) data.data = cb->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) cb->cb(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) list_del(&task->list_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void cmdq_task_handle_error(struct cmdq_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct cmdq_thread *thread = task->thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct cmdq_task *next_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct cmdq *cmdq = task->cmdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) dev_err(cmdq->mbox.dev, "task 0x%p error\n", task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) next_task = list_first_entry_or_null(&thread->task_busy_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct cmdq_task, list_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (next_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) writel(next_task->pa_base >> cmdq->shift_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) thread->base + CMDQ_THR_CURR_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) cmdq_thread_resume(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static void cmdq_thread_irq_handler(struct cmdq *cmdq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct cmdq_thread *thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct cmdq_task *task, *tmp, *curr_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) u32 curr_pa, irq_flag, task_end_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) bool err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) irq_flag = readl(thread->base + CMDQ_THR_IRQ_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) writel(~irq_flag, thread->base + CMDQ_THR_IRQ_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * When ISR call this function, another CPU core could run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * "release task" right before we acquire the spin lock, and thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * reset / disable this GCE thread, so we need to check the enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * bit of this GCE thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!(readl(thread->base + CMDQ_THR_ENABLE_TASK) & CMDQ_THR_ENABLED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (irq_flag & CMDQ_THR_IRQ_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) err = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) else if (irq_flag & CMDQ_THR_IRQ_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) err = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) << cmdq->shift_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) list_for_each_entry_safe(task, tmp, &thread->task_busy_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) list_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) task_end_pa = task->pa_base + task->pkt->cmd_buf_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (curr_pa >= task->pa_base && curr_pa < task_end_pa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) curr_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (!curr_task || curr_pa == task_end_pa - CMDQ_INST_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) cmdq_task_exec_done(task, CMDQ_CB_NORMAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) kfree(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) } else if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) cmdq_task_exec_done(task, CMDQ_CB_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) cmdq_task_handle_error(curr_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) kfree(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (curr_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (list_empty(&thread->task_busy_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) cmdq_thread_disable(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) clk_disable(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static irqreturn_t cmdq_irq_handler(int irq, void *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct cmdq *cmdq = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) unsigned long irq_status, flags = 0L;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) int bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) irq_status = readl(cmdq->base + CMDQ_CURR_IRQ_STATUS) & cmdq->irq_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (!(irq_status ^ cmdq->irq_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) for_each_clear_bit(bit, &irq_status, cmdq->thread_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct cmdq_thread *thread = &cmdq->thread[bit];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) spin_lock_irqsave(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) cmdq_thread_irq_handler(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) spin_unlock_irqrestore(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static int cmdq_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct cmdq *cmdq = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct cmdq_thread *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) bool task_running = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) cmdq->suspended = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) for (i = 0; i < cmdq->thread_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) thread = &cmdq->thread[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!list_empty(&thread->task_busy_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) task_running = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (task_running)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) dev_warn(dev, "exist running task(s) in suspend\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) clk_unprepare(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static int cmdq_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct cmdq *cmdq = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) WARN_ON(clk_prepare(cmdq->clock) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) cmdq->suspended = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static int cmdq_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct cmdq *cmdq = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) clk_unprepare(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct cmdq_pkt *pkt = (struct cmdq_pkt *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct cmdq_task *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) unsigned long curr_pa, end_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /* Client should not flush new tasks if suspended. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) WARN_ON(cmdq->suspended);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) task = kzalloc(sizeof(*task), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (!task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) task->cmdq = cmdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) INIT_LIST_HEAD(&task->list_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) task->pa_base = pkt->pa_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) task->thread = thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) task->pkt = pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (list_empty(&thread->task_busy_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) WARN_ON(clk_enable(cmdq->clock) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * The thread reset will clear thread related register to 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * including pc, end, priority, irq, suspend and enable. Thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * set CMDQ_THR_ENABLED to CMDQ_THR_ENABLE_TASK will enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * thread and make it running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) WARN_ON(cmdq_thread_reset(cmdq, thread) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) writel(task->pa_base >> cmdq->shift_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) thread->base + CMDQ_THR_CURR_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->shift_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) thread->base + CMDQ_THR_END_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) writel(thread->priority, thread->base + CMDQ_THR_PRIORITY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) writel(CMDQ_THR_IRQ_EN, thread->base + CMDQ_THR_IRQ_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) writel(CMDQ_THR_ENABLED, thread->base + CMDQ_THR_ENABLE_TASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) cmdq->shift_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) end_pa = readl(thread->base + CMDQ_THR_END_ADDR) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) cmdq->shift_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) /* check boundary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (curr_pa == end_pa - CMDQ_INST_SIZE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) curr_pa == end_pa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* set to this task directly */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) writel(task->pa_base >> cmdq->shift_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) thread->base + CMDQ_THR_CURR_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) cmdq_task_insert_into_thread(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) smp_mb(); /* modify jump before enable thread */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->shift_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) thread->base + CMDQ_THR_END_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) cmdq_thread_resume(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) list_move_tail(&task->list_entry, &thread->task_busy_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static int cmdq_mbox_startup(struct mbox_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void cmdq_mbox_shutdown(struct mbox_chan *chan)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) struct cmdq_task *task, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) spin_lock_irqsave(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (list_empty(&thread->task_busy_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) /* make sure executed tasks have success callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) cmdq_thread_irq_handler(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (list_empty(&thread->task_busy_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) list_for_each_entry_safe(task, tmp, &thread->task_busy_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) list_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) cmdq_task_exec_done(task, CMDQ_CB_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) kfree(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) cmdq_thread_disable(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) clk_disable(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * The thread->task_busy_list empty means thread already disable. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * cmdq_mbox_send_data() always reset thread which clear disable and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * suspend statue when first pkt send to channel, so there is no need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * to do any operation here, only unlock and leave.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) spin_unlock_irqrestore(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static int cmdq_mbox_flush(struct mbox_chan *chan, unsigned long timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct cmdq_task_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) struct cmdq_cb_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct cmdq_task *task, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) u32 enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) spin_lock_irqsave(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (list_empty(&thread->task_busy_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (!cmdq_thread_is_in_wfe(thread))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) goto wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) list_for_each_entry_safe(task, tmp, &thread->task_busy_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) list_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) cb = &task->pkt->async_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (cb->cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) data.sta = CMDQ_CB_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) data.data = cb->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) cb->cb(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) list_del(&task->list_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) kfree(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) cmdq_thread_resume(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) cmdq_thread_disable(cmdq, thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) clk_disable(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) spin_unlock_irqrestore(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) wait:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) cmdq_thread_resume(thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) spin_unlock_irqrestore(&thread->chan->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (readl_poll_timeout_atomic(thread->base + CMDQ_THR_ENABLE_TASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) enable, enable == 0, 1, timeout)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) dev_err(cmdq->mbox.dev, "Fail to wait GCE thread 0x%x done\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) (u32)(thread->base - cmdq->base));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static const struct mbox_chan_ops cmdq_mbox_chan_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) .send_data = cmdq_mbox_send_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) .startup = cmdq_mbox_startup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) .shutdown = cmdq_mbox_shutdown,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) .flush = cmdq_mbox_flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static struct mbox_chan *cmdq_xlate(struct mbox_controller *mbox,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) const struct of_phandle_args *sp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) int ind = sp->args[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct cmdq_thread *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (ind >= mbox->num_chans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) thread = (struct cmdq_thread *)mbox->chans[ind].con_priv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) thread->priority = sp->args[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) thread->chan = &mbox->chans[ind];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return &mbox->chans[ind];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) static int cmdq_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) struct cmdq *cmdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) int err, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct gce_plat *plat_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) cmdq = devm_kzalloc(dev, sizeof(*cmdq), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (!cmdq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) cmdq->base = devm_ioremap_resource(dev, res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if (IS_ERR(cmdq->base)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) dev_err(dev, "failed to ioremap gce\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return PTR_ERR(cmdq->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) cmdq->irq = platform_get_irq(pdev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (cmdq->irq < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) return cmdq->irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) plat_data = (struct gce_plat *)of_device_get_match_data(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (!plat_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) dev_err(dev, "failed to get match data\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) cmdq->thread_nr = plat_data->thread_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) cmdq->shift_pa = plat_data->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) cmdq->irq_mask = GENMASK(cmdq->thread_nr - 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) err = devm_request_irq(dev, cmdq->irq, cmdq_irq_handler, IRQF_SHARED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) "mtk_cmdq", cmdq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) dev_err(dev, "failed to register ISR (%d)\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) dev_dbg(dev, "cmdq device: addr:0x%p, va:0x%p, irq:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) dev, cmdq->base, cmdq->irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) cmdq->clock = devm_clk_get(dev, "gce");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (IS_ERR(cmdq->clock)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) dev_err(dev, "failed to get gce clk\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) return PTR_ERR(cmdq->clock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) cmdq->mbox.dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) cmdq->mbox.chans = devm_kcalloc(dev, cmdq->thread_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) sizeof(*cmdq->mbox.chans), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (!cmdq->mbox.chans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) cmdq->mbox.num_chans = cmdq->thread_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) cmdq->mbox.ops = &cmdq_mbox_chan_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) cmdq->mbox.of_xlate = cmdq_xlate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /* make use of TXDONE_BY_ACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) cmdq->mbox.txdone_irq = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) cmdq->mbox.txdone_poll = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) cmdq->thread = devm_kcalloc(dev, cmdq->thread_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) sizeof(*cmdq->thread), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (!cmdq->thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) for (i = 0; i < cmdq->thread_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) cmdq->thread[i].base = cmdq->base + CMDQ_THR_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) CMDQ_THR_SIZE * i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) INIT_LIST_HEAD(&cmdq->thread[i].task_busy_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) cmdq->mbox.chans[i].con_priv = (void *)&cmdq->thread[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) err = devm_mbox_controller_register(dev, &cmdq->mbox);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) dev_err(dev, "failed to register mailbox: %d\n", err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) platform_set_drvdata(pdev, cmdq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) WARN_ON(clk_prepare(cmdq->clock) < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) cmdq_init(cmdq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static const struct dev_pm_ops cmdq_pm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) .suspend = cmdq_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) .resume = cmdq_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static const struct gce_plat gce_plat_v2 = {.thread_nr = 16};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static const struct gce_plat gce_plat_v3 = {.thread_nr = 24};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static const struct gce_plat gce_plat_v4 = {.thread_nr = 24, .shift = 3};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) static const struct of_device_id cmdq_of_ids[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) {.compatible = "mediatek,mt8173-gce", .data = (void *)&gce_plat_v2},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) {.compatible = "mediatek,mt8183-gce", .data = (void *)&gce_plat_v3},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {.compatible = "mediatek,mt6779-gce", .data = (void *)&gce_plat_v4},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static struct platform_driver cmdq_drv = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) .probe = cmdq_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) .remove = cmdq_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) .name = "mtk_cmdq",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) .pm = &cmdq_pm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) .of_match_table = cmdq_of_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) static int __init cmdq_drv_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return platform_driver_register(&cmdq_drv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) static void __exit cmdq_drv_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) platform_driver_unregister(&cmdq_drv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) subsys_initcall(cmdq_drv_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) module_exit(cmdq_drv_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) MODULE_LICENSE("GPL v2");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags