^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) Rockchip Electronics Co.Ltd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author: Felix Zeng <felix.zeng@rock-chips.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/sync_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "rknpu_ioctl.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "rknpu_drv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "rknpu_reset.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "rknpu_gem.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "rknpu_fence.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "rknpu_job.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "rknpu_mem.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define _REG_READ(base, offset) readl(base + (offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define _REG_WRITE(base, value, offset) writel(value, base + (offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define REG_READ(offset) _REG_READ(rknpu_core_base, offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define REG_WRITE(value, offset) _REG_WRITE(rknpu_core_base, value, offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static int rknpu_core_index(int core_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) int index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) if (core_mask & RKNPU_CORE0_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) else if (core_mask & RKNPU_CORE1_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) index = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) else if (core_mask & RKNPU_CORE2_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) index = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int rknpu_core_mask(int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) int core_mask = RKNPU_CORE_AUTO_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) switch (core_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) core_mask = RKNPU_CORE0_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) core_mask = RKNPU_CORE1_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) core_mask = RKNPU_CORE2_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return core_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static int rknn_get_task_number(struct rknpu_job *job, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int task_num = job->args->task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (job->use_core_num == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) task_num = job->args->subcore_task[core_index].task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) else if (job->use_core_num == 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) task_num = job->args->subcore_task[core_index + 2].task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return task_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static void rknpu_job_free(struct rknpu_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct rknpu_gem_object *task_obj = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) task_obj =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) (struct rknpu_gem_object *)(uintptr_t)job->args->task_obj_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (task_obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) rknpu_gem_object_put(&task_obj->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (job->fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) dma_fence_put(job->fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (job->args_owner)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) kfree(job->args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) kfree(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static int rknpu_job_cleanup(struct rknpu_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) rknpu_job_free(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static void rknpu_job_cleanup_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct rknpu_job *job =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) container_of(work, struct rknpu_job, cleanup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) rknpu_job_cleanup(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct rknpu_submit *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct rknpu_gem_object *task_obj = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (rknpu_dev->config->num_irqs == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) args->core_mask = RKNPU_CORE0_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) job = kzalloc(sizeof(*job), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (!job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) job->timestamp = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) job->rknpu_dev = rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) job->use_core_num = (args->core_mask & RKNPU_CORE0_MASK) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) ((args->core_mask & RKNPU_CORE1_MASK) >> 1) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) ((args->core_mask & RKNPU_CORE2_MASK) >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) job->run_count = job->use_core_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) job->interrupt_count = job->use_core_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (task_obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) rknpu_gem_object_get(&task_obj->base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (!(args->flags & RKNPU_JOB_NONBLOCK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) job->args = args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) job->args_owner = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) job->args = kzalloc(sizeof(*args), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!job->args) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) kfree(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) *job->args = *args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) job->args_owner = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) INIT_WORK(&job->cleanup_work, rknpu_job_cleanup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static inline int rknpu_job_wait(struct rknpu_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct rknpu_submit *args = job->args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct rknpu_task *last_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) void __iomem *rknpu_core_base = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int core_index = rknpu_core_index(job->args->core_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) int wait_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) subcore_data = &rknpu_dev->subcore_datas[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ret = wait_event_interruptible_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) subcore_data->job_done_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) job->flags & RKNPU_JOB_DONE || rknpu_dev->soft_reseting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) msecs_to_jiffies(args->timeout));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (++wait_count >= 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) } while (ret == 0 && job->in_queue[core_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (job->in_queue[core_index]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) spin_lock_irqsave(&rknpu_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) list_del_init(&job->head[core_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) subcore_data->task_num -= rknn_get_task_number(job, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) job->in_queue[core_index] = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) spin_unlock_irqrestore(&rknpu_dev->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return ret < 0 ? ret : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) last_task = job->last_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (!last_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return ret < 0 ? ret : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) last_task->int_status = job->int_status[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (ret <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) args->task_counter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) rknpu_core_base = rknpu_dev->base[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (args->flags & RKNPU_JOB_PC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) uint32_t task_status =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) REG_READ(RKNPU_OFFSET_PC_TASK_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) args->task_counter =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) (task_status &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) rknpu_dev->config->pc_task_number_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) "failed to wait job, task counter: %d, flags: %#x, ret = %d, elapsed time: %lldus\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) args->task_counter, args->flags, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return ret < 0 ? ret : -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (!(job->flags & RKNPU_JOB_DONE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) args->task_counter = args->task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct rknpu_submit *args = job->args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct rknpu_gem_object *task_obj =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct rknpu_mem_object *task_obj =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) (struct rknpu_mem_object *)(uintptr_t)args->task_obj_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct rknpu_task *task_base = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct rknpu_task *first_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct rknpu_task *last_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) int task_start = args->task_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) int task_end = args->task_start + args->task_number - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) int task_number = args->task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) int pc_data_amount_scale = rknpu_dev->config->pc_data_amount_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) int pc_task_number_bits = rknpu_dev->config->pc_task_number_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (!task_obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (rknpu_dev->config->num_irqs > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (i == core_index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) REG_WRITE((0xe + 0x10000000 * i), 0x1004);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) REG_WRITE((0xe + 0x10000000 * i), 0x3004);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (job->use_core_num == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) task_start = args->subcore_task[core_index].task_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) task_end = args->subcore_task[core_index].task_start +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) args->subcore_task[core_index].task_number -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) task_number =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) args->subcore_task[core_index].task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) } else if (job->use_core_num == 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) task_start = args->subcore_task[core_index].task_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) task_end = args->subcore_task[core_index].task_start +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) args->subcore_task[core_index].task_number -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) task_number =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) args->subcore_task[core_index].task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) } else if (job->use_core_num == 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) task_start =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) args->subcore_task[core_index + 2].task_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) task_end =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) args->subcore_task[core_index + 2].task_start +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) args->subcore_task[core_index + 2].task_number -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) task_number =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) args->subcore_task[core_index + 2].task_number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) task_base = task_obj->kv_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) first_task = &task_base[task_start];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) last_task = &task_base[task_end];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) REG_WRITE((first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) pc_data_amount_scale - 1) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) pc_data_amount_scale -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) RKNPU_OFFSET_PC_DATA_AMOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) REG_WRITE(last_task->int_mask, RKNPU_OFFSET_INT_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) REG_WRITE(first_task->int_mask, RKNPU_OFFSET_INT_CLEAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) REG_WRITE(((0x6 | task_pp_en) << pc_task_number_bits) | task_number,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) RKNPU_OFFSET_PC_TASK_CONTROL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) REG_WRITE(args->task_base_addr, RKNPU_OFFSET_PC_DMA_BASE_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) job->first_task = first_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) job->last_task = last_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) job->int_mask[core_index] = last_task->int_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) REG_WRITE(0x1, RKNPU_OFFSET_PC_OP_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) REG_WRITE(0x0, RKNPU_OFFSET_PC_OP_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static int rknpu_job_commit(struct rknpu_job *job, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct rknpu_submit *args = job->args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) // switch to slave mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (!(args->flags & RKNPU_JOB_PC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return rknpu_job_commit_pc(job, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (rknpu_dev->soft_reseting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) subcore_data = &rknpu_dev->subcore_datas[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (subcore_data->job || list_empty(&subcore_data->todo_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) job = list_first_entry(&subcore_data->todo_list, struct rknpu_job,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) head[core_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) list_del_init(&job->head[core_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) job->in_queue[core_index] = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) subcore_data->job = job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) job->run_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) job->hw_recoder_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (job->run_count == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (job->args->core_mask & RKNPU_CORE0_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) job->ret = rknpu_job_commit(job, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (job->args->core_mask & RKNPU_CORE1_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) job->ret = rknpu_job_commit(job, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (job->args->core_mask & RKNPU_CORE2_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) job->ret = rknpu_job_commit(job, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) subcore_data = &rknpu_dev->subcore_datas[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) subcore_data->job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) subcore_data->task_num -= rknn_get_task_number(job, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) job->interrupt_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) subcore_data->timer.busy_time +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ktime_us_delta(now, job->hw_recoder_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (job->interrupt_count == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) int use_core_num = job->use_core_num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) job->flags |= RKNPU_JOB_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) job->ret = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (job->fence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) dma_fence_signal(job->fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (job->flags & RKNPU_JOB_ASYNC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) schedule_work(&job->cleanup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (use_core_num > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) wake_up(&(&rknpu_dev->subcore_datas[0])->job_done_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) wake_up(&subcore_data->job_done_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) rknpu_job_next(rknpu_dev, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) static void rknpu_job_schedule(struct rknpu_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int i = 0, core_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) int task_num_list[3] = { 0, 1, 2 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) int tmp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if ((job->args->core_mask & 0x07) == RKNPU_CORE_AUTO_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (rknpu_dev->subcore_datas[0].task_num >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) rknpu_dev->subcore_datas[1].task_num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) tmp = task_num_list[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) task_num_list[1] = task_num_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) task_num_list[0] = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (rknpu_dev->subcore_datas[task_num_list[0]].task_num >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) rknpu_dev->subcore_datas[2].task_num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) tmp = task_num_list[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) task_num_list[2] = task_num_list[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) task_num_list[1] = task_num_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) task_num_list[0] = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) } else if (rknpu_dev->subcore_datas[task_num_list[1]].task_num >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) rknpu_dev->subcore_datas[2].task_num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) tmp = task_num_list[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) task_num_list[2] = task_num_list[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) task_num_list[1] = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!rknpu_dev->subcore_datas[task_num_list[0]].job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) core_index = task_num_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) else if (!rknpu_dev->subcore_datas[task_num_list[1]].job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) core_index = task_num_list[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) else if (!rknpu_dev->subcore_datas[task_num_list[2]].job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) core_index = task_num_list[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) core_index = task_num_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) job->args->core_mask = rknpu_core_mask(core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) job->use_core_num = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) job->interrupt_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) job->run_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (job->args->core_mask & rknpu_core_mask(i)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) subcore_data = &rknpu_dev->subcore_datas[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) list_add_tail(&job->head[i], &subcore_data->todo_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) subcore_data->task_num += rknn_get_task_number(job, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) job->in_queue[i] = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (job->args->core_mask & rknpu_core_mask(i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) rknpu_job_next(rknpu_dev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static void rknpu_job_abort(struct rknpu_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct rknpu_device *rknpu_dev = job->rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) int core_index = rknpu_core_index(job->args->core_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) msleep(100);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) if (job->args->core_mask & rknpu_core_mask(i)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) subcore_data = &rknpu_dev->subcore_datas[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (job == subcore_data->job && !job->irq_entry[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) subcore_data->job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) subcore_data->task_num -=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) rknn_get_task_number(job, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (job->ret == -ETIMEDOUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) "job timeout, flags: %#x, irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x, elapsed time: %lldus\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) job->flags, REG_READ(RKNPU_OFFSET_INT_STATUS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) job->int_mask[core_index],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) (REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) rknpu_dev->config->pc_task_number_mask),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) rknpu_soft_reset(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) "job abort, flags: %#x, ret: %d, elapsed time: %lldus\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) job->flags, job->ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) ktime_to_us(ktime_sub(ktime_get(), job->timestamp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) rknpu_job_cleanup(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static inline uint32_t rknpu_fuzz_status(uint32_t status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) uint32_t fuzz_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if ((status & 0x3) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) fuzz_status |= 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if ((status & 0xc) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) fuzz_status |= 0xc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if ((status & 0x30) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) fuzz_status |= 0x30;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if ((status & 0xc0) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) fuzz_status |= 0xc0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if ((status & 0x300) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) fuzz_status |= 0x300;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if ((status & 0xc00) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) fuzz_status |= 0xc00;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return fuzz_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) static inline irqreturn_t rknpu_irq_handler(int irq, void *data, int core_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) struct rknpu_device *rknpu_dev = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) void __iomem *rknpu_core_base = rknpu_dev->base[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) uint32_t status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) subcore_data = &rknpu_dev->subcore_datas[core_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) job = subcore_data->job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (!job) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) rknpu_job_next(rknpu_dev, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) job->irq_entry[core_index] = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) status = REG_READ(RKNPU_OFFSET_INT_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) job->int_status[core_index] = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (rknpu_fuzz_status(status) != job->int_mask[core_index]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) "invalid irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) status, REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) job->int_mask[core_index],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) (REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) rknpu_dev->config->pc_task_number_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) rknpu_job_done(job, 0, core_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) irqreturn_t rknpu_core0_irq_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return rknpu_irq_handler(irq, data, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) irqreturn_t rknpu_core1_irq_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return rknpu_irq_handler(irq, data, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) irqreturn_t rknpu_core2_irq_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return rknpu_irq_handler(irq, data, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) int core_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (core_mask & rknpu_core_mask(i)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) subcore_data = &rknpu_dev->subcore_datas[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) job = subcore_data->job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (job &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) ktime_to_ms(ktime_sub(now, job->timestamp)) >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) job->args->timeout) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) rknpu_soft_reset(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) subcore_data->job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) spin_unlock_irqrestore(&rknpu_dev->irq_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) schedule_work(&job->cleanup_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) spin_lock_irqsave(&rknpu_dev->irq_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (!list_empty(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) &subcore_data->todo_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) job = list_first_entry(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) &subcore_data->todo_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) struct rknpu_job,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) head[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) list_del_init(&job->head[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) job->in_queue[i] = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) spin_unlock_irqrestore(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) &rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) } while (job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) static int rknpu_submit(struct rknpu_device *rknpu_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct rknpu_submit *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (args->task_number == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) LOG_ERROR("invalid rknpu task number!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) job = rknpu_job_alloc(rknpu_dev, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (!job) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) LOG_ERROR("failed to allocate rknpu job!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (args->flags & RKNPU_JOB_FENCE_IN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) #ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) struct dma_fence *in_fence;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) in_fence = sync_file_get_fence(args->fence_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (!in_fence) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) LOG_ERROR("invalid fence in fd, fd: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) args->fence_fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) args->fence_fd = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * Wait if the fence is from a foreign context, or if the fence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * array contains any fence from a foreign context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (!dma_fence_match_context(in_fence,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) rknpu_dev->fence_ctx->context))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ret = dma_fence_wait_timeout(in_fence, true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) args->timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) dma_fence_put(in_fence);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (ret != -ERESTARTSYS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) LOG_ERROR("Error (%d) waiting for fence!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) "failed to use rknpu fence, please enable rknpu fence config!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) rknpu_job_free(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (args->flags & RKNPU_JOB_FENCE_OUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) #ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) ret = rknpu_fence_alloc(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) rknpu_job_free(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) job->args->fence_fd = rknpu_fence_get_fd(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) args->fence_fd = job->args->fence_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) LOG_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) "failed to use rknpu fence, please enable rknpu fence config!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) rknpu_job_free(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (args->flags & RKNPU_JOB_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) job->flags |= RKNPU_JOB_ASYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) rknpu_job_timeout_clean(rknpu_dev, job->args->core_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) rknpu_job_schedule(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) ret = job->ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) rknpu_job_abort(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) rknpu_job_schedule(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (args->flags & RKNPU_JOB_PC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) job->ret = rknpu_job_wait(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) args->task_counter = job->args->task_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) ret = job->ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) rknpu_job_cleanup(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) rknpu_job_abort(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) int rknpu_submit_ioctl(struct drm_device *dev, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) struct drm_file *file_priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) struct rknpu_submit *args = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return rknpu_submit(rknpu_dev, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) int rknpu_submit_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct rknpu_submit args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) if (unlikely(copy_from_user(&args, (struct rknpu_submit *)data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) sizeof(struct rknpu_submit)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) LOG_ERROR("%s: copy_from_user failed\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) ret = rknpu_submit(rknpu_dev, &args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (unlikely(copy_to_user((struct rknpu_submit *)data, &args,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) sizeof(struct rknpu_submit)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) LOG_ERROR("%s: copy_to_user failed\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) void __iomem *rknpu_core_base = rknpu_dev->base[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (version == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) *version = REG_READ(RKNPU_OFFSET_VERSION) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) REG_READ(RKNPU_OFFSET_VERSION_NUM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) uint32_t *expect, uint32_t *tw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) void __iomem *base = rknpu_dev->bw_priority_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (!rknpu_dev->config->bw_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) LOG_WARN("Get bw_priority is not supported on this device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (!base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) spin_lock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (priority != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) *priority = _REG_READ(base, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) if (expect != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) *expect = _REG_READ(base, 0x8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (tw != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) *tw = _REG_READ(base, 0xc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) spin_unlock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) uint32_t expect, uint32_t tw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) void __iomem *base = rknpu_dev->bw_priority_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (!rknpu_dev->config->bw_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) LOG_WARN("Set bw_priority is not supported on this device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (!base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) spin_lock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (priority != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) _REG_WRITE(base, priority, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (expect != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) _REG_WRITE(base, expect, 0x8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) if (tw != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) _REG_WRITE(base, tw, 0xc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) spin_unlock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) void __iomem *rknpu_core_base = rknpu_dev->base[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (!rknpu_dev->config->bw_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) LOG_WARN("Clear rw_amount is not supported on this device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) spin_lock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) spin_unlock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) uint32_t *dt_rd, uint32_t *wd_rd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) void __iomem *rknpu_core_base = rknpu_dev->base[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) int amount_scale = rknpu_dev->config->pc_data_amount_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (!rknpu_dev->config->bw_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) LOG_WARN("Get rw_amount is not supported on this device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) spin_lock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (dt_wr != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) *dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT) * amount_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (dt_rd != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) *dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT) * amount_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (wd_rd != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) *wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT) * amount_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) spin_unlock(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) uint32_t dt_wr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) uint32_t dt_rd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) uint32_t wd_rd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (!rknpu_dev->config->bw_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) LOG_WARN(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) "Get total_rw_amount is not supported on this device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) ret = rknpu_get_rw_amount(rknpu_dev, &dt_wr, &dt_rd, &wd_rd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) if (amount != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) *amount = dt_wr + dt_rd + wd_rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags