^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) Rockchip Electronics Co.Ltd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author: Felix Zeng <felix.zeng@rock-chips.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/dma-buf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/irqdomain.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/iopoll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/of_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/of_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/of_platform.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/of_reserved_mem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/printk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/time.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/clk-provider.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/pm_domain.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/devfreq_cooling.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/regmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/dma-iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/of_address.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <soc/rockchip/rockchip_iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <soc/rockchip/rockchip_opp_select.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <soc/rockchip/rockchip_system_monitor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <soc/rockchip/rockchip_ipa.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <../drivers/devfreq/governor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include "rknpu_ioctl.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include "rknpu_reset.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include "rknpu_fence.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "rknpu_drv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include "rknpu_gem.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <drm/drm_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <drm/drm_ioctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <drm/drm_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <drm/drm_drv.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "rknpu_gem.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <linux/rk-dma-heap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include "rknpu_mem.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define POWER_DOWN_FREQ 200000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define NPU_MMU_DISABLED_POLL_PERIOD_US 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define NPU_MMU_DISABLED_POLL_TIMEOUT_US 20000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static int bypass_irq_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) module_param(bypass_irq_handler, int, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) MODULE_PARM_DESC(bypass_irq_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) "bypass RKNPU irq handler if set it to 1, disabled by default");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static int bypass_soft_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) module_param(bypass_soft_reset, int, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) MODULE_PARM_DESC(bypass_soft_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) "bypass RKNPU soft reset if set it to 1, disabled by default");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct npu_irqs_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) irqreturn_t (*irq_hdl)(int irq, void *ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static const struct npu_irqs_data rk356x_npu_irqs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) { "npu_irq", rknpu_core0_irq_handler }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static const struct npu_irqs_data rk3588_npu_irqs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) { "npu0_irq", rknpu_core0_irq_handler },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) { "npu1_irq", rknpu_core1_irq_handler },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) { "npu2_irq", rknpu_core2_irq_handler }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static const struct npu_irqs_data rv110x_npu_irqs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) { "npu_irq", rknpu_core0_irq_handler }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static const struct npu_reset_data rk356x_npu_resets[] = { { "srst_a",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) "srst_h" } };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static const struct npu_reset_data rk3588_npu_resets[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) { "srst_a0", "srst_h0" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) { "srst_a1", "srst_h1" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) { "srst_a2", "srst_h2" }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static const struct npu_reset_data rv110x_npu_resets[] = { { "srst_a",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) "srst_h" } };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static const struct rknpu_config rk356x_rknpu_config = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) .bw_priority_addr = 0xfe180008,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) .bw_priority_length = 0x10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) .dma_mask = DMA_BIT_MASK(32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) .pc_data_amount_scale = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) .pc_task_number_bits = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) .pc_task_number_mask = 0xfff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .bw_enable = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) .irqs = rk356x_npu_irqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) .resets = rk356x_npu_resets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) .num_irqs = ARRAY_SIZE(rk356x_npu_irqs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) .num_resets = ARRAY_SIZE(rk356x_npu_resets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static const struct rknpu_config rk3588_rknpu_config = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) .bw_priority_addr = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) .bw_priority_length = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) .dma_mask = DMA_BIT_MASK(40),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) .pc_data_amount_scale = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) .pc_task_number_bits = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) .pc_task_number_mask = 0xfff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) .bw_enable = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) .irqs = rk3588_npu_irqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) .resets = rk3588_npu_resets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) .num_irqs = ARRAY_SIZE(rk3588_npu_irqs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) .num_resets = ARRAY_SIZE(rk3588_npu_resets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static const struct rknpu_config rv1106_rknpu_config = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .bw_priority_addr = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .bw_priority_length = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .dma_mask = DMA_BIT_MASK(32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) .pc_data_amount_scale = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) .pc_task_number_bits = 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .pc_task_number_mask = 0xffff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .bw_enable = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .irqs = rv110x_npu_irqs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .resets = rv110x_npu_resets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) .num_irqs = ARRAY_SIZE(rv110x_npu_irqs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .num_resets = ARRAY_SIZE(rv110x_npu_resets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* driver probe and init */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static const struct of_device_id rknpu_of_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .compatible = "rockchip,rknpu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .data = &rk356x_rknpu_config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .compatible = "rockchip,rk3568-rknpu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .data = &rk356x_rknpu_config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .compatible = "rockchip,rk3588-rknpu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) .data = &rk3588_rknpu_config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) .compatible = "rockchip,rv1106-rknpu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) .data = &rv1106_rknpu_config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static int rknpu_get_drv_version(uint32_t *version)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) *version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) DRIVER_PATCHLEVEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static int rknpu_power_on(struct rknpu_device *rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static int rknpu_power_off(struct rknpu_device *rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void rknpu_power_off_delay_work(struct work_struct *power_off_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct rknpu_device *rknpu_dev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) container_of(to_delayed_work(power_off_work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct rknpu_device, power_off_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) mutex_lock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) rknpu_power_off(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) mutex_unlock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int rknpu_power_get(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) cancel_delayed_work(&rknpu_dev->power_off_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) mutex_lock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (atomic_inc_return(&rknpu_dev->power_refcount) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) ret = rknpu_power_on(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) mutex_unlock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int rknpu_power_put(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) mutex_lock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (atomic_dec_if_positive(&rknpu_dev->power_refcount) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) ret = rknpu_power_off(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) mutex_unlock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static int rknpu_power_put_delay(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) mutex_lock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (atomic_read(&rknpu_dev->power_refcount) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) queue_delayed_work(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) rknpu_dev->power_off_wq, &rknpu_dev->power_off_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) msecs_to_jiffies(rknpu_dev->power_put_delay));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) atomic_dec_if_positive(&rknpu_dev->power_refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) mutex_unlock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static int rknpu_action(struct rknpu_device *rknpu_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct rknpu_action *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) switch (args->flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) case RKNPU_GET_HW_VERSION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ret = rknpu_get_hw_version(rknpu_dev, &args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) case RKNPU_GET_DRV_VERSION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) ret = rknpu_get_drv_version(&args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) case RKNPU_GET_FREQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) args->value = clk_get_rate(rknpu_dev->clks[0].clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) case RKNPU_SET_FREQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) case RKNPU_GET_VOLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) args->value = regulator_get_voltage(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) case RKNPU_SET_VOLT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) case RKNPU_ACT_RESET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) ret = rknpu_soft_reset(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) case RKNPU_GET_BW_PRIORITY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) case RKNPU_SET_BW_PRIORITY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) case RKNPU_GET_BW_EXPECT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) case RKNPU_SET_BW_EXPECT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) case RKNPU_GET_BW_TW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) &args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) case RKNPU_SET_BW_TW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ret = rknpu_clear_rw_amount(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) case RKNPU_GET_DT_WR_AMOUNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) case RKNPU_GET_DT_RD_AMOUNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) case RKNPU_GET_WT_RD_AMOUNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) case RKNPU_GET_TOTAL_RW_AMOUNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) case RKNPU_GET_IOMMU_EN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) args->value = rknpu_dev->iommu_en;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) case RKNPU_SET_PROC_NICE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) set_user_nice(current, *(int32_t *)&args->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) case RKNPU_GET_TOTAL_SRAM_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (rknpu_dev->sram_mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) args->value = rknpu_dev->sram_mm->total_chunks *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) rknpu_dev->sram_mm->chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) args->value = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) case RKNPU_GET_FREE_SRAM_SIZE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (rknpu_dev->sram_mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) args->value = rknpu_dev->sram_mm->free_chunks *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) rknpu_dev->sram_mm->chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) args->value = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static int rknpu_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return nonseekable_open(inode, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) static int rknpu_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) unsigned long data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct rknpu_action args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (unlikely(copy_from_user(&args, (struct rknpu_action *)data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) sizeof(struct rknpu_action)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) LOG_ERROR("%s: copy_from_user failed\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) ret = rknpu_action(rknpu_dev, &args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (unlikely(copy_to_user((struct rknpu_action *)data, &args,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) sizeof(struct rknpu_action)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) LOG_ERROR("%s: copy_to_user failed\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) long ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct rknpu_device *rknpu_dev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) container_of(file->private_data, struct rknpu_device, miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) rknpu_power_get(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) case IOCTL_RKNPU_ACTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) ret = rknpu_action_ioctl(rknpu_dev, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) case IOCTL_RKNPU_SUBMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ret = rknpu_submit_ioctl(rknpu_dev, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) case IOCTL_RKNPU_MEM_CREATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) ret = rknpu_mem_create_ioctl(rknpu_dev, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) case RKNPU_MEM_MAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) case IOCTL_RKNPU_MEM_DESTROY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) case IOCTL_RKNPU_MEM_SYNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) ret = rknpu_mem_sync_ioctl(rknpu_dev, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) rknpu_power_put_delay(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) const struct file_operations rknpu_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) .open = rknpu_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) .release = rknpu_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) .unlocked_ioctl = rknpu_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) .compat_ioctl = rknpu_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static const struct vm_operations_struct rknpu_gem_vm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) .fault = rknpu_gem_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) .open = drm_gem_vm_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) .close = drm_gem_vm_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static int rknpu_action_ioctl(struct drm_device *dev, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct drm_file *file_priv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) return rknpu_action(rknpu_dev, (struct rknpu_action *)data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) #define RKNPU_IOCTL(func) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) static int __##func(struct drm_device *dev, void *data, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct drm_file *file_priv) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) int ret = -EINVAL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) rknpu_power_get(rknpu_dev); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) ret = func(dev, data, file_priv); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) rknpu_power_put_delay(rknpu_dev); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) RKNPU_IOCTL(rknpu_action_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) RKNPU_IOCTL(rknpu_submit_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) RKNPU_IOCTL(rknpu_gem_create_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) RKNPU_IOCTL(rknpu_gem_map_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) RKNPU_IOCTL(rknpu_gem_destroy_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) RKNPU_IOCTL(rknpu_gem_sync_ioctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static const struct drm_ioctl_desc rknpu_ioctls[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) DRM_IOCTL_DEF_DRV(RKNPU_ACTION, __rknpu_action_ioctl, DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, __rknpu_submit_ioctl, DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, __rknpu_gem_create_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, __rknpu_gem_map_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, __rknpu_gem_destroy_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, __rknpu_gem_sync_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) DRM_RENDER_ALLOW),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static const struct file_operations rknpu_drm_driver_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) .open = drm_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) .mmap = rknpu_gem_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) .poll = drm_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) .read = drm_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) .unlocked_ioctl = drm_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) .compat_ioctl = drm_compat_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) .release = drm_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static struct drm_driver rknpu_drm_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) #if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) .driver_features = DRIVER_GEM | DRIVER_RENDER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) .gem_free_object_unlocked = rknpu_gem_free_object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) .gem_vm_ops = &rknpu_gem_vm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) .dumb_create = rknpu_gem_dumb_create,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) #if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) .dumb_map_offset = rknpu_gem_dumb_map_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) .dumb_map_offset = drm_gem_dumb_map_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) .dumb_destroy = drm_gem_dumb_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) .gem_prime_export = drm_gem_prime_export,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) #if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) .gem_prime_import = rknpu_gem_prime_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) .gem_prime_import = drm_gem_prime_import,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) .gem_prime_vmap = rknpu_gem_prime_vmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) .gem_prime_vunmap = rknpu_gem_prime_vunmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) .gem_prime_mmap = rknpu_gem_prime_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) .ioctls = rknpu_ioctls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) .num_ioctls = ARRAY_SIZE(rknpu_ioctls),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) .fops = &rknpu_drm_driver_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) .name = DRIVER_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) .desc = DRIVER_DESC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) .date = DRIVER_DATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) .major = DRIVER_MAJOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) .minor = DRIVER_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) .patchlevel = DRIVER_PATCHLEVEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static enum hrtimer_restart hrtimer_handler(struct hrtimer *timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct rknpu_device *rknpu_dev =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) container_of(timer, struct rknpu_device, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct rknpu_subcore_data *subcore_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct rknpu_job *job = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) ktime_t now = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) subcore_data = &rknpu_dev->subcore_datas[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) job = subcore_data->job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (job) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) subcore_data->timer.busy_time +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) ktime_us_delta(now, job->hw_recoder_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) job->hw_recoder_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) subcore_data->timer.busy_time_record =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) subcore_data->timer.busy_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) subcore_data->timer.busy_time = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) hrtimer_forward_now(timer, rknpu_dev->kt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static void rknpu_init_timer(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) rknpu_dev->kt = ktime_set(0, RKNPU_LOAD_INTERVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) hrtimer_init(&rknpu_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) rknpu_dev->timer.function = hrtimer_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) hrtimer_start(&rknpu_dev->timer, rknpu_dev->kt, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static void rknpu_cancel_timer(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) hrtimer_cancel(&rknpu_dev->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static bool rknpu_is_iommu_enable(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct device_node *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) iommu = of_parse_phandle(dev->of_node, "iommus", 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (!iommu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) LOG_DEV_INFO(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) "rknpu iommu device-tree entry not found!, using non-iommu mode\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (!of_device_is_available(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) LOG_DEV_INFO(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) "rknpu iommu is disabled, using non-iommu mode\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) of_node_put(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) of_node_put(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct drm_device *drm_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (IS_ERR(drm_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return PTR_ERR(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /* register the DRM device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) ret = drm_dev_register(drm_dev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) goto err_free_drm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) drm_dev->dev_private = rknpu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) rknpu_dev->drm_dev = drm_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) err_free_drm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) drm_dev_put(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) drm_dev_unref(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct drm_device *drm_dev = rknpu_dev->drm_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) drm_dev_unregister(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) #if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) drm_dev_put(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) drm_dev_unref(drm_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) static int rknpu_power_on(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) if (rknpu_dev->vdd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) ret = regulator_enable(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) "failed to enable vdd reg for rknpu, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (rknpu_dev->mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ret = regulator_enable(rknpu_dev->mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) "failed to enable mem reg for rknpu, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) if (rknpu_dev->multiple_domains) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (rknpu_dev->genpd_dev_npu0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) ret = pm_runtime_resume_and_get(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) rknpu_dev->genpd_dev_npu0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) "failed to get pm runtime for npu0, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (rknpu_dev->genpd_dev_npu1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) ret = pm_runtime_resume_and_get(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) rknpu_dev->genpd_dev_npu1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) "failed to get pm runtime for npu1, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) if (rknpu_dev->genpd_dev_npu2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) ret = pm_runtime_resume_and_get(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) rknpu_dev->genpd_dev_npu2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) ret = pm_runtime_get_sync(rknpu_dev->genpd_dev_npu2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) "failed to get pm runtime for npu2, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) ret = pm_runtime_get_sync(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) "failed to get pm runtime for rknpu, ret: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) static int rknpu_power_off(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) bool val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) pm_runtime_put_sync(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (rknpu_dev->multiple_domains) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * Because IOMMU's runtime suspend callback is asynchronous,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * So it may be executed after the NPU is turned off after PD/CLK/VD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * and the runtime suspend callback has a register access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * If the PD/VD/CLK is closed, the register access will crash.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * As a workaround, it's safe to close pd stuff until iommu disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * If pm runtime framework can handle this issue in the future, remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) ret = readx_poll_timeout(rockchip_iommu_is_enabled, dev, val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) !val, NPU_MMU_DISABLED_POLL_PERIOD_US,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) NPU_MMU_DISABLED_POLL_TIMEOUT_US);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) LOG_DEV_ERROR(dev, "iommu still enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) pm_runtime_get_sync(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) rockchip_monitor_volt_adjust_unlock(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (rknpu_dev->iommu_en)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) msleep(20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (rknpu_dev->genpd_dev_npu2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (rknpu_dev->genpd_dev_npu1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (rknpu_dev->genpd_dev_npu0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) pm_runtime_put_sync(rknpu_dev->genpd_dev_npu0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (rknpu_dev->vdd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) regulator_disable(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (rknpu_dev->mem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) regulator_disable(rknpu_dev->mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) static struct monitor_dev_profile npu_mdevp = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) .type = MONITOR_TPYE_DEV,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) .update_volt = rockchip_monitor_check_rate_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) static int npu_opp_helper(struct dev_pm_set_opp_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) struct device *dev = data->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) struct dev_pm_opp_supply *old_supply_vdd = &data->old_opp.supplies[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct dev_pm_opp_supply *old_supply_mem = &data->old_opp.supplies[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) struct dev_pm_opp_supply *new_supply_vdd = &data->new_opp.supplies[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) struct dev_pm_opp_supply *new_supply_mem = &data->new_opp.supplies[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) struct regulator *vdd_reg = data->regulators[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) struct regulator *mem_reg = data->regulators[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct clk *clk = data->clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) unsigned long old_freq = data->old_opp.rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) unsigned long new_freq = data->new_opp.rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) bool is_set_rm = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) bool is_set_clk = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) u32 target_rm = UINT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (!pm_runtime_active(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) is_set_rm = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (opp_info->scmi_clk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) is_set_clk = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) rockchip_get_read_margin(dev, opp_info, new_supply_vdd->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) &target_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /* Change frequency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) LOG_DEV_DEBUG(dev, "switching OPP: %lu Hz --> %lu Hz\n", old_freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) new_freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /* Scaling up? Scale voltage before frequency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) if (new_freq >= old_freq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) new_freq, true, is_set_clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) "failed to set volt %lu uV for mem reg\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) new_supply_mem->u_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) goto restore_voltage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) "failed to set volt %lu uV for vdd reg\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) new_supply_vdd->u_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) goto restore_voltage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (is_set_clk && clk_set_rate(clk, new_freq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) goto restore_rm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) /* Scaling down? Scale voltage after frequency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) rockchip_set_intermediate_rate(dev, opp_info, clk, old_freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) new_freq, false, is_set_clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) rockchip_set_read_margin(dev, opp_info, target_rm, is_set_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (is_set_clk && clk_set_rate(clk, new_freq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) LOG_DEV_ERROR(dev, "failed to set clk rate: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) goto restore_rm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) ret = regulator_set_voltage(vdd_reg, new_supply_vdd->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) "failed to set volt %lu uV for vdd reg\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) new_supply_vdd->u_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) goto restore_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) ret = regulator_set_voltage(mem_reg, new_supply_mem->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) "failed to set volt %lu uV for mem reg\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) new_supply_mem->u_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) goto restore_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) restore_freq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) if (is_set_clk && clk_set_rate(clk, old_freq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) LOG_DEV_ERROR(dev, "failed to restore old-freq %lu Hz\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) old_freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) restore_rm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) rockchip_get_read_margin(dev, opp_info, old_supply_vdd->u_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) &target_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) rockchip_set_read_margin(dev, opp_info, opp_info->current_rm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) is_set_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) restore_voltage:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) regulator_set_voltage(mem_reg, old_supply_mem->u_volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) regulator_set_voltage(vdd_reg, old_supply_vdd->u_volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) static int npu_devfreq_target(struct device *dev, unsigned long *freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct dev_pm_opp *opp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) unsigned long opp_volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (!npu_mdevp.is_checked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) opp = devfreq_recommended_opp(dev, freq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) if (IS_ERR(opp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) return PTR_ERR(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) opp_volt = dev_pm_opp_get_voltage(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) dev_pm_opp_put(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) rockchip_monitor_volt_adjust_lock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) ret = dev_pm_opp_set_rate(dev, *freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) rknpu_dev->current_freq = *freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) if (rknpu_dev->devfreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) rknpu_dev->devfreq->last_status.current_frequency =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) *freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) rknpu_dev->current_volt = opp_volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) rknpu_dev->current_freq, rknpu_dev->current_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) rockchip_monitor_volt_adjust_unlock(rknpu_dev->mdev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) struct dev_pm_opp *opp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) unsigned long freq = *target_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) unsigned long old_freq = rknpu_dev->current_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) unsigned long volt, old_volt = rknpu_dev->current_volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) opp = devfreq_recommended_opp(dev, &freq, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (IS_ERR(opp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) return PTR_ERR(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) volt = dev_pm_opp_get_voltage(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * Only update if there is a change of frequency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (old_freq == freq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) *target_freq = freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) if (old_volt == volt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) rknpu_dev->current_volt = volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) freq, volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) *target_freq = freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) rknpu_dev->current_freq = freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (rknpu_dev->devfreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) rknpu_dev->devfreq->last_status.current_frequency = freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) rknpu_dev->current_volt = volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) LOG_DEV_INFO(dev, "set rknpu freq: %lu, volt: %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) rknpu_dev->current_freq, rknpu_dev->current_volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) static int npu_devfreq_get_dev_status(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) struct devfreq_dev_status *stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) *freq = rknpu_dev->current_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) static struct devfreq_dev_profile npu_devfreq_profile = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) .polling_ms = 50,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) .target = npu_devfreq_target,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) .get_dev_status = npu_devfreq_get_dev_status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) .get_cur_freq = npu_devfreq_get_cur_freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) static int devfreq_rknpu_ondemand_func(struct devfreq *df, unsigned long *freq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) struct rknpu_device *rknpu_dev = df->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) *freq = rknpu_dev->ondemand_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *freq = df->previous_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) static int devfreq_rknpu_ondemand_handler(struct devfreq *devfreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) unsigned int event, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) static struct devfreq_governor devfreq_rknpu_ondemand = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) .name = "rknpu_ondemand",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) .get_target_freq = devfreq_rknpu_ondemand_func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) .event_handler = devfreq_rknpu_ondemand_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) static unsigned long npu_get_static_power(struct devfreq *devfreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) unsigned long voltage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) struct device *dev = devfreq->dev.parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) if (!rknpu_dev->model_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) static struct devfreq_cooling_power npu_cooling_power = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) .get_static_power = &npu_get_static_power,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) static int rk3588_npu_set_read_margin(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) struct rockchip_opp_info *opp_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) u32 rm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) u32 offset = 0, val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) int i, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) if (!opp_info->grf || !opp_info->volt_rm_tbl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) if (rm == opp_info->current_rm || rm == UINT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) for (i = 0; i < 3; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) ret = regmap_read(opp_info->grf, offset, &val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) val &= ~0x1c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) regmap_write(opp_info->grf, offset, val | (rm << 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) offset += 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) opp_info->current_rm = rm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) static const struct rockchip_opp_data rk3588_npu_opp_data = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) .set_read_margin = rk3588_npu_set_read_margin,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) static const struct of_device_id rockchip_npu_of_match[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) .compatible = "rockchip,rk3588",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) .data = (void *)&rk3588_npu_opp_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) struct devfreq_dev_profile *dp = &npu_devfreq_profile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) struct dev_pm_opp *opp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) struct opp_table *reg_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) struct opp_table *opp_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) const char *const reg_names[] = { "rknpu", "mem" };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (of_find_property(dev->of_node, "rknpu-supply", NULL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) of_find_property(dev->of_node, "mem-supply", NULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) reg_table = dev_pm_opp_set_regulators(dev, reg_names, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) if (IS_ERR(reg_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) return PTR_ERR(reg_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) opp_table =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) dev_pm_opp_register_set_opp_helper(dev, npu_opp_helper);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (IS_ERR(opp_table)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) dev_pm_opp_put_regulators(reg_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) return PTR_ERR(opp_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) reg_table = dev_pm_opp_set_regulators(dev, reg_names, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (IS_ERR(reg_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) return PTR_ERR(reg_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) rockchip_get_opp_data(rockchip_npu_of_match, &rknpu_dev->opp_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) ret = rockchip_init_opp_table(dev, &rknpu_dev->opp_info, "npu_leakage",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) "rknpu");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) opp = devfreq_recommended_opp(dev, &rknpu_dev->current_freq, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) if (IS_ERR(opp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) ret = PTR_ERR(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) goto err_remove_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) dev_pm_opp_put(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) dp->initial_freq = rknpu_dev->current_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) goto err_remove_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) (void *)rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) if (IS_ERR(rknpu_dev->devfreq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) LOG_DEV_ERROR(dev, "failed to add devfreq\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) ret = PTR_ERR(rknpu_dev->devfreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) goto err_remove_governor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) rknpu_dev->devfreq->last_status.total_time = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) rknpu_dev->devfreq->last_status.busy_time = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) npu_mdevp.data = rknpu_dev->devfreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) npu_mdevp.opp_info = &rknpu_dev->opp_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) rknpu_dev->mdev_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) rockchip_system_monitor_register(dev, &npu_mdevp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (IS_ERR(rknpu_dev->mdev_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) LOG_DEV_DEBUG(dev, "without system monitor\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) rknpu_dev->mdev_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) npu_mdevp.is_checked = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) (u32 *)&npu_cooling_power.dyn_power_coeff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) rknpu_dev->model_data =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) rockchip_ipa_power_model_init(dev, "npu_leakage");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) rknpu_dev->model_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) LOG_DEV_ERROR(dev, "failed to initialize power model\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) } else if (rknpu_dev->model_data->dynamic_coefficient) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) npu_cooling_power.dyn_power_coeff =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) rknpu_dev->model_data->dynamic_coefficient;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) if (!npu_cooling_power.dyn_power_coeff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) LOG_DEV_ERROR(dev, "failed to register cooling device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) err_remove_governor:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) devfreq_remove_governor(&devfreq_rknpu_ondemand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) err_remove_table:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) dev_pm_opp_of_remove_table(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) rknpu_dev->devfreq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) unsigned long volt, old_freq, freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) struct dev_pm_opp *opp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) freq = old_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) opp = devfreq_recommended_opp(dev, &freq, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) volt = dev_pm_opp_get_voltage(opp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) #if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) if (freq >= old_freq && rknpu_dev->vdd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) if (freq < old_freq && rknpu_dev->vdd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) rknpu_dev->current_freq = freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) rknpu_dev->current_volt = volt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) struct devfreq_dev_profile *dp = &npu_devfreq_profile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) goto err_remove_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) dp->initial_freq = rknpu_dev->current_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) ret = devfreq_add_governor(&devfreq_rknpu_ondemand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) LOG_DEV_ERROR(dev, "failed to add rknpu_ondemand governor\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) goto err_remove_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) rknpu_dev->devfreq = devm_devfreq_add_device(dev, dp, "rknpu_ondemand",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) (void *)rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) if (IS_ERR(rknpu_dev->devfreq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) LOG_DEV_ERROR(dev, "failed to add devfreq\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) ret = PTR_ERR(rknpu_dev->devfreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) goto err_remove_governor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) rknpu_dev->devfreq->last_status.total_time = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) rknpu_dev->devfreq->last_status.busy_time = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) npu_mdevp.data = rknpu_dev->devfreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) rknpu_dev->mdev_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) rockchip_system_monitor_register(dev, &npu_mdevp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (IS_ERR(rknpu_dev->mdev_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) LOG_DEV_DEBUG(dev, "without system monitor\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) rknpu_dev->mdev_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) rknpu_dev->current_freq = clk_get_rate(rknpu_dev->clks[0].clk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) rknpu_dev->current_volt = regulator_get_voltage(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) (u32 *)&npu_cooling_power.dyn_power_coeff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) rknpu_dev->model_data =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) rockchip_ipa_power_model_init(dev, "npu_leakage");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) rknpu_dev->model_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) LOG_DEV_ERROR(dev, "failed to initialize power model\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) } else if (rknpu_dev->model_data->dynamic_coefficient) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) npu_cooling_power.dyn_power_coeff =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) rknpu_dev->model_data->dynamic_coefficient;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if (!npu_cooling_power.dyn_power_coeff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) LOG_DEV_ERROR(dev, "failed to register cooling device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) err_remove_governor:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) devfreq_remove_governor(&devfreq_rknpu_ondemand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) err_remove_table:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) dev_pm_opp_of_remove_table(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) rknpu_dev->devfreq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) static int rknpu_devfreq_remove(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (rknpu_dev->devfreq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) devfreq_unregister_opp_notifier(rknpu_dev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) rknpu_dev->devfreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) dev_pm_opp_of_remove_table(rknpu_dev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) #ifdef CONFIG_PM_DEVFREQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) devfreq_remove_governor(&devfreq_rknpu_ondemand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) static int rknpu_register_irq(struct platform_device *pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) const struct rknpu_config *config = rknpu_dev->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) int i, ret, irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) res = platform_get_resource_byname(pdev, IORESOURCE_IRQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) config->irqs[0].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) /* there are irq names in dts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) for (i = 0; i < config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) irq = platform_get_irq_byname(pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) config->irqs[i].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) if (irq < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) LOG_DEV_ERROR(dev, "no npu %s in dts\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) config->irqs[i].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) return irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) ret = devm_request_irq(dev, irq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) config->irqs[i].irq_hdl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) IRQF_SHARED, dev_name(dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) LOG_DEV_ERROR(dev, "request %s failed: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) config->irqs[i].name, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) /* no irq names in dts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) irq = platform_get_irq(pdev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) if (irq < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) LOG_DEV_ERROR(dev, "no npu irq in dts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) return irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) ret = devm_request_irq(dev, irq, rknpu_core0_irq_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) IRQF_SHARED, dev_name(dev), rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) LOG_DEV_ERROR(dev, "request irq failed: %d\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) static int rknpu_find_sram_resource(struct rknpu_device *rknpu_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) struct device *dev = rknpu_dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) struct device_node *sram_node = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) struct resource sram_res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) uint32_t sram_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) /* get sram device node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) sram_node = of_parse_phandle(dev->of_node, "rockchip,sram", 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) rknpu_dev->sram_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) if (!sram_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) /* get sram start and size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) ret = of_address_to_resource(sram_node, 0, &sram_res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) of_node_put(sram_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) /* check sram start and size is PAGE_SIZE align */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) rknpu_dev->sram_start = round_up(sram_res.start, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) rknpu_dev->sram_end = round_down(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) sram_res.start + resource_size(&sram_res), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (rknpu_dev->sram_end <= rknpu_dev->sram_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) LOG_DEV_WARN(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) "invalid sram resource, sram start %pa, sram end %pa\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) &rknpu_dev->sram_start, &rknpu_dev->sram_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) sram_size = rknpu_dev->sram_end - rknpu_dev->sram_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) rknpu_dev->sram_base_io =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) devm_ioremap(dev, rknpu_dev->sram_start, sram_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (IS_ERR(rknpu_dev->sram_base_io)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) LOG_DEV_ERROR(dev, "failed to remap sram base io!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) rknpu_dev->sram_base_io = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) rknpu_dev->sram_size = sram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) LOG_DEV_INFO(dev, "sram region: [%pa, %pa), sram size: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) &rknpu_dev->sram_start, &rknpu_dev->sram_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) rknpu_dev->sram_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) static int rknpu_probe(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) struct resource *res = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) struct rknpu_device *rknpu_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) struct device *virt_dev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) const struct of_device_id *match = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) const struct rknpu_config *config = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) int ret = -EINVAL, i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (!pdev->dev.of_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) match = of_match_device(rknpu_of_match, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) if (!match) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) if (!rknpu_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) config = of_device_get_match_data(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) if (!config)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) rknpu_dev->config = config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) rknpu_dev->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) if (!rknpu_dev->iommu_en) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) /* Initialize reserved memory resources */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) ret = of_reserved_mem_device_init(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) LOG_DEV_INFO(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) "initialize reserved memory for rknpu device!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) rknpu_dev->bypass_irq_handler = bypass_irq_handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) rknpu_dev->bypass_soft_reset = bypass_soft_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) rknpu_reset_get(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) if (rknpu_dev->num_clks < 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) if (strstr(__clk_get_name(rknpu_dev->clks[0].clk), "scmi"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) rknpu_dev->opp_info.scmi_clk = rknpu_dev->clks[0].clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) if (IS_ERR(rknpu_dev->vdd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) ret = PTR_ERR(rknpu_dev->vdd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) "failed to get vdd regulator for rknpu: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) rknpu_dev->vdd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) rknpu_dev->mem = devm_regulator_get_optional(dev, "mem");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) if (IS_ERR(rknpu_dev->mem)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) if (PTR_ERR(rknpu_dev->mem) != -ENODEV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) ret = PTR_ERR(rknpu_dev->mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) "failed to get mem regulator for rknpu: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) rknpu_dev->mem = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) spin_lock_init(&rknpu_dev->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) spin_lock_init(&rknpu_dev->irq_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) mutex_init(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) mutex_init(&rknpu_dev->reset_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) for (i = 0; i < config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) rknpu_dev->subcore_datas[i].task_num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) res = platform_get_resource(pdev, IORESOURCE_MEM, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) if (!res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) "failed to get memory resource for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) rknpu_dev->base[i] = devm_ioremap_resource(dev, res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) if (PTR_ERR(rknpu_dev->base[i]) == -EBUSY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) rknpu_dev->base[i] = devm_ioremap(dev, res->start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) resource_size(res));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) if (IS_ERR(rknpu_dev->base[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) "failed to remap register for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) return PTR_ERR(rknpu_dev->base[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) if (config->bw_priority_length > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) rknpu_dev->bw_priority_base =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) devm_ioremap(dev, config->bw_priority_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) config->bw_priority_length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) if (IS_ERR(rknpu_dev->bw_priority_base)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) LOG_DEV_ERROR(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) rknpu_dev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) "failed to remap bw priority register for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) rknpu_dev->bw_priority_base = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) if (!rknpu_dev->bypass_irq_handler) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) ret = rknpu_register_irq(pdev, rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) LOG_DEV_WARN(dev, "bypass irq handler!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) ret = rknpu_drm_probe(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) rknpu_dev->miscdev.minor = MISC_DYNAMIC_MINOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) rknpu_dev->miscdev.name = "rknpu";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) rknpu_dev->miscdev.fops = &rknpu_fops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) ret = misc_register(&rknpu_dev->miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) LOG_DEV_ERROR(dev, "cannot register miscdev (%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) rknpu_dev->heap = rk_dma_heap_find("rk-dma-heap-cma");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) if (!rknpu_dev->heap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) LOG_DEV_ERROR(dev, "failed to find cma heap\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) rk_dma_heap_set_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) LOG_DEV_INFO(dev, "Initialized %s: v%d.%d.%d for %s\n", DRIVER_DESC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) DRIVER_MAJOR, DRIVER_MINOR, DRIVER_PATCHLEVEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) DRIVER_DATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) #ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) ret = rknpu_fence_context_alloc(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) LOG_DEV_ERROR(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) "failed to allocate fence context for rknpu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) goto err_remove_drv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) platform_set_drvdata(pdev, rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) pm_runtime_enable(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (of_count_phandle_with_args(dev->of_node, "power-domains",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) "#power-domain-cells") > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) virt_dev = dev_pm_domain_attach_by_name(dev, "npu0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) if (!IS_ERR(virt_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) rknpu_dev->genpd_dev_npu0 = virt_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) virt_dev = dev_pm_domain_attach_by_name(dev, "npu1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) if (!IS_ERR(virt_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) rknpu_dev->genpd_dev_npu1 = virt_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) virt_dev = dev_pm_domain_attach_by_name(dev, "npu2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) if (!IS_ERR(virt_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) rknpu_dev->genpd_dev_npu2 = virt_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) rknpu_dev->multiple_domains = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) ret = rknpu_power_on(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) goto err_remove_drv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) rknpu_devfreq_init(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) // set default power put delay to 3s
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) rknpu_dev->power_put_delay = 3000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) rknpu_dev->power_off_wq =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) create_freezable_workqueue("rknpu_power_off_wq");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (!rknpu_dev->power_off_wq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) LOG_DEV_ERROR(dev, "rknpu couldn't create power_off workqueue");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) goto err_devfreq_remove;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) INIT_DEFERRABLE_WORK(&rknpu_dev->power_off_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) rknpu_power_off_delay_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->iommu_en) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) if (!rknpu_find_sram_resource(rknpu_dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) ret = rknpu_mm_create(rknpu_dev->sram_size, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) &rknpu_dev->sram_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) goto err_remove_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) LOG_DEV_WARN(dev, "could not find sram resource!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) rknpu_power_off(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) atomic_set(&rknpu_dev->power_refcount, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) atomic_set(&rknpu_dev->cmdline_power_refcount, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) rknpu_debugger_init(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) rknpu_init_timer(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) err_remove_wq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) destroy_workqueue(rknpu_dev->power_off_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) err_devfreq_remove:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) rknpu_devfreq_remove(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) err_remove_drv:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) rknpu_drm_remove(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) misc_deregister(&(rknpu_dev->miscdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) static int rknpu_remove(struct platform_device *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) cancel_delayed_work_sync(&rknpu_dev->power_off_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) destroy_workqueue(rknpu_dev->power_off_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) rknpu_mm_destroy(rknpu_dev->sram_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) rknpu_debugger_remove(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) rknpu_cancel_timer(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) WARN_ON(rknpu_dev->subcore_datas[i].job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) rknpu_drm_remove(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) misc_deregister(&(rknpu_dev->miscdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) rknpu_devfreq_remove(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) mutex_lock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) if (atomic_read(&rknpu_dev->power_refcount) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) rknpu_power_off(rknpu_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) mutex_unlock(&rknpu_dev->power_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) if (rknpu_dev->multiple_domains) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) if (rknpu_dev->genpd_dev_npu0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu0, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) if (rknpu_dev->genpd_dev_npu1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu1, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) if (rknpu_dev->genpd_dev_npu2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) dev_pm_domain_detach(rknpu_dev->genpd_dev_npu2, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) pm_runtime_disable(&pdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) #if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) static int rknpu_runtime_suspend(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) if (opp_info->scmi_clk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) if (clk_set_rate(opp_info->scmi_clk, POWER_DOWN_FREQ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) LOG_DEV_ERROR(dev, "failed to restore clk rate\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) opp_info->current_rm = UINT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) static int rknpu_runtime_resume(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) struct rockchip_opp_info *opp_info = &rknpu_dev->opp_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) if (!rknpu_dev->current_freq || !rknpu_dev->current_volt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) ret = clk_bulk_prepare_enable(opp_info->num_clks, opp_info->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) LOG_DEV_ERROR(dev, "failed to enable opp clks\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) if (opp_info->data && opp_info->data->set_read_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) opp_info->data->set_read_margin(dev, opp_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) opp_info->target_rm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) if (opp_info->scmi_clk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) if (clk_set_rate(opp_info->scmi_clk, rknpu_dev->current_freq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) LOG_DEV_ERROR(dev, "failed to set power down rate\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) clk_bulk_disable_unprepare(opp_info->num_clks, opp_info->clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) static const struct dev_pm_ops rknpu_pm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) pm_runtime_force_resume)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) SET_RUNTIME_PM_OPS(rknpu_runtime_suspend, rknpu_runtime_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) static struct platform_driver rknpu_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) .probe = rknpu_probe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) .remove = rknpu_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) .driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) .name = "RKNPU",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) #ifndef FPGA_PLATFORM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) #if KERNEL_VERSION(5, 5, 0) < LINUX_VERSION_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) .pm = &rknpu_pm_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) .of_match_table = of_match_ptr(rknpu_of_match),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) static int rknpu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) return platform_driver_register(&rknpu_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) static void rknpu_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) platform_driver_unregister(&rknpu_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) late_initcall(rknpu_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) module_exit(rknpu_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) MODULE_DESCRIPTION("RKNPU driver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) MODULE_AUTHOR("Felix Zeng <felix.zeng@rock-chips.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) MODULE_ALIAS("rockchip-rknpu");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) DRIVER_PATCHLEVEL));
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags