^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright 2016-2019 HabanaLabs, Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "goyaP.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "../include/hw_ip/mmu/mmu_general.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "../include/hw_ip/mmu/mmu_v1_0.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "../include/goya/asic_reg/goya_masks.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "../include/goya/goya_reg_map.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/genalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/hwmon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/io-64-nonatomic-lo-hi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * GOYA security scheme:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * 1. Host is protected by:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * - Range registers (When MMU is enabled, DMA RR does NOT protect host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * 2. DRAM is protected by:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * - Range registers (protect the first 512MB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - MMU (isolation between users)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * 3. Configuration is protected by:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * - Range registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * - Protection bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * When MMU is disabled:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * QMAN DMA: PQ, CQ, CP, DMA are secured.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * PQ, CB and the data are on the host.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * QMAN TPC/MME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * PQ, CQ and CP are not secured.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * PQ, CB and the data are on the SRAM/DRAM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * Since QMAN DMA is secured, the driver is parsing the DMA CB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * - checks DMA pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * - WREG, MSG_PROT are not allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * - MSG_LONG/SHORT are allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * A read/write transaction by the QMAN to a protected area will succeed if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * and only if the QMAN's CP is secured and MSG_PROT is used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * When MMU is enabled:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * QMAN DMA: PQ, CQ and CP are secured.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * MMU is set to bypass on the Secure props register of the QMAN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * The reasons we don't enable MMU for PQ, CQ and CP are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * - PQ entry is in kernel address space and the driver doesn't map it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * - CP writes to MSIX register and to kernel address space (completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * queue).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * DMA is not secured but because CP is secured, the driver still needs to parse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * the CB, but doesn't need to check the DMA addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * For QMAN DMA 0, DMA is also secured because only the driver uses this DMA and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * the driver doesn't map memory in MMU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * QMAN TPC/MME: PQ, CQ and CP aren't secured (no change from MMU disabled mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * DMA RR does NOT protect host because DMA is not secured
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define GOYA_BOOT_FIT_FILE "habanalabs/goya/goya-boot-fit.itb"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define GOYA_LINUX_FW_FILE "habanalabs/goya/goya-fit.itb"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define GOYA_MMU_REGS_NUM 63
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define GOYA_DMA_POOL_BLK_SIZE 0x100 /* 256 bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define GOYA_RESET_TIMEOUT_MSEC 500 /* 500ms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define GOYA_PLDM_RESET_TIMEOUT_MSEC 20000 /* 20s */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define GOYA_RESET_WAIT_MSEC 1 /* 1ms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define GOYA_CPU_RESET_WAIT_MSEC 100 /* 100ms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define GOYA_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define GOYA_TEST_QUEUE_WAIT_USEC 100000 /* 100ms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define GOYA_PLDM_MMU_TIMEOUT_USEC (MMU_CONFIG_TIMEOUT_USEC * 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define GOYA_QMAN0_FENCE_VAL 0xD169B243
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define GOYA_MAX_STRING_LEN 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define GOYA_CB_POOL_CB_CNT 512
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define GOYA_CB_POOL_CB_SIZE 0x20000 /* 128KB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define IS_QM_IDLE(engine, qm_glbl_sts0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) (((qm_glbl_sts0) & engine##_QM_IDLE_MASK) == engine##_QM_IDLE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define IS_DMA_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(DMA, qm_glbl_sts0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define IS_TPC_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(TPC, qm_glbl_sts0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define IS_MME_QM_IDLE(qm_glbl_sts0) IS_QM_IDLE(MME, qm_glbl_sts0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define IS_CMDQ_IDLE(engine, cmdq_glbl_sts0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) (((cmdq_glbl_sts0) & engine##_CMDQ_IDLE_MASK) == \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) engine##_CMDQ_IDLE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) IS_CMDQ_IDLE(TPC, cmdq_glbl_sts0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) IS_CMDQ_IDLE(MME, cmdq_glbl_sts0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define IS_DMA_IDLE(dma_core_sts0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) !((dma_core_sts0) & DMA_CH_0_STS0_DMA_BUSY_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define IS_TPC_IDLE(tpc_cfg_sts) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) (((tpc_cfg_sts) & TPC_CFG_IDLE_MASK) == TPC_CFG_IDLE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define IS_MME_IDLE(mme_arch_sts) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) (((mme_arch_sts) & MME_ARCH_IDLE_MASK) == MME_ARCH_IDLE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static const char goya_irq_name[GOYA_MSIX_ENTRIES][GOYA_MAX_STRING_LEN] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) "goya cq 0", "goya cq 1", "goya cq 2", "goya cq 3",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) "goya cq 4", "goya cpu eq"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static u16 goya_packet_sizes[MAX_PACKET_ID] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) [PACKET_WREG_32] = sizeof(struct packet_wreg32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) [PACKET_WREG_BULK] = sizeof(struct packet_wreg_bulk),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) [PACKET_MSG_LONG] = sizeof(struct packet_msg_long),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) [PACKET_MSG_SHORT] = sizeof(struct packet_msg_short),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) [PACKET_CP_DMA] = sizeof(struct packet_cp_dma),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) [PACKET_MSG_PROT] = sizeof(struct packet_msg_prot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) [PACKET_FENCE] = sizeof(struct packet_fence),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) [PACKET_LIN_DMA] = sizeof(struct packet_lin_dma),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) [PACKET_NOP] = sizeof(struct packet_nop),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) [PACKET_STOP] = sizeof(struct packet_stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static inline bool validate_packet_id(enum packet_id id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) switch (id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) case PACKET_WREG_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) case PACKET_WREG_BULK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) case PACKET_MSG_LONG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) case PACKET_MSG_SHORT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) case PACKET_CP_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) case PACKET_MSG_PROT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) case PACKET_FENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) case PACKET_LIN_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) case PACKET_NOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) case PACKET_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static u64 goya_mmu_regs[GOYA_MMU_REGS_NUM] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) mmDMA_QM_0_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) mmDMA_QM_1_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) mmDMA_QM_2_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) mmDMA_QM_3_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) mmDMA_QM_4_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) mmTPC0_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) mmTPC0_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) mmTPC0_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) mmTPC0_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) mmTPC0_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) mmTPC0_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) mmTPC1_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) mmTPC1_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) mmTPC1_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) mmTPC1_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) mmTPC1_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) mmTPC1_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) mmTPC2_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) mmTPC2_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) mmTPC2_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) mmTPC2_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) mmTPC2_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) mmTPC2_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) mmTPC3_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) mmTPC3_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) mmTPC3_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) mmTPC3_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) mmTPC3_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) mmTPC3_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) mmTPC4_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) mmTPC4_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) mmTPC4_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) mmTPC4_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) mmTPC4_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) mmTPC4_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) mmTPC5_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mmTPC5_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) mmTPC5_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) mmTPC5_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) mmTPC5_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) mmTPC5_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) mmTPC6_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) mmTPC6_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) mmTPC6_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) mmTPC6_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) mmTPC6_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) mmTPC6_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) mmTPC7_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) mmTPC7_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) mmTPC7_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) mmTPC7_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) mmTPC7_CFG_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) mmTPC7_CFG_AWUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) mmMME_QM_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) mmMME_QM_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) mmMME_CMDQ_GLBL_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) mmMME_CMDQ_GLBL_NON_SECURE_PROPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) mmMME_SBA_CONTROL_DATA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) mmMME_SBB_CONTROL_DATA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) mmMME_SBC_CONTROL_DATA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) mmMME_WBC_CONTROL_DATA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) mmPCIE_WRAP_PSOC_ARUSER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) mmPCIE_WRAP_PSOC_AWUSER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static u32 goya_all_events[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) GOYA_ASYNC_EVENT_ID_PCIE_IF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) GOYA_ASYNC_EVENT_ID_TPC0_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) GOYA_ASYNC_EVENT_ID_TPC1_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) GOYA_ASYNC_EVENT_ID_TPC2_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) GOYA_ASYNC_EVENT_ID_TPC3_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) GOYA_ASYNC_EVENT_ID_TPC4_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) GOYA_ASYNC_EVENT_ID_TPC5_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) GOYA_ASYNC_EVENT_ID_TPC6_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) GOYA_ASYNC_EVENT_ID_TPC7_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) GOYA_ASYNC_EVENT_ID_MME_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) GOYA_ASYNC_EVENT_ID_MME_ECC_EXT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) GOYA_ASYNC_EVENT_ID_MMU_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) GOYA_ASYNC_EVENT_ID_DMA_MACRO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) GOYA_ASYNC_EVENT_ID_DMA_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) GOYA_ASYNC_EVENT_ID_CPU_IF_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) GOYA_ASYNC_EVENT_ID_PSOC_MEM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) GOYA_ASYNC_EVENT_ID_SRAM0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) GOYA_ASYNC_EVENT_ID_SRAM1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) GOYA_ASYNC_EVENT_ID_SRAM2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) GOYA_ASYNC_EVENT_ID_SRAM3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) GOYA_ASYNC_EVENT_ID_SRAM4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) GOYA_ASYNC_EVENT_ID_SRAM5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) GOYA_ASYNC_EVENT_ID_SRAM6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) GOYA_ASYNC_EVENT_ID_SRAM7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) GOYA_ASYNC_EVENT_ID_SRAM8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) GOYA_ASYNC_EVENT_ID_SRAM9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) GOYA_ASYNC_EVENT_ID_SRAM10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) GOYA_ASYNC_EVENT_ID_SRAM11,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) GOYA_ASYNC_EVENT_ID_SRAM12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) GOYA_ASYNC_EVENT_ID_SRAM13,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) GOYA_ASYNC_EVENT_ID_SRAM14,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) GOYA_ASYNC_EVENT_ID_SRAM15,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) GOYA_ASYNC_EVENT_ID_SRAM16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) GOYA_ASYNC_EVENT_ID_SRAM17,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) GOYA_ASYNC_EVENT_ID_SRAM18,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) GOYA_ASYNC_EVENT_ID_SRAM19,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) GOYA_ASYNC_EVENT_ID_SRAM20,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) GOYA_ASYNC_EVENT_ID_SRAM21,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) GOYA_ASYNC_EVENT_ID_SRAM22,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) GOYA_ASYNC_EVENT_ID_SRAM23,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) GOYA_ASYNC_EVENT_ID_SRAM24,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) GOYA_ASYNC_EVENT_ID_SRAM25,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) GOYA_ASYNC_EVENT_ID_SRAM26,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) GOYA_ASYNC_EVENT_ID_SRAM27,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) GOYA_ASYNC_EVENT_ID_SRAM28,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) GOYA_ASYNC_EVENT_ID_SRAM29,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) GOYA_ASYNC_EVENT_ID_GIC500,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) GOYA_ASYNC_EVENT_ID_PLL0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) GOYA_ASYNC_EVENT_ID_PLL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) GOYA_ASYNC_EVENT_ID_PLL3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) GOYA_ASYNC_EVENT_ID_PLL4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) GOYA_ASYNC_EVENT_ID_PLL5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) GOYA_ASYNC_EVENT_ID_PLL6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) GOYA_ASYNC_EVENT_ID_AXI_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) GOYA_ASYNC_EVENT_ID_L2_RAM_ECC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) GOYA_ASYNC_EVENT_ID_PCIE_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) GOYA_ASYNC_EVENT_ID_TPC0_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) GOYA_ASYNC_EVENT_ID_TPC1_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) GOYA_ASYNC_EVENT_ID_TPC2_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) GOYA_ASYNC_EVENT_ID_TPC3_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) GOYA_ASYNC_EVENT_ID_TPC4_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) GOYA_ASYNC_EVENT_ID_TPC5_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) GOYA_ASYNC_EVENT_ID_TPC6_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) GOYA_ASYNC_EVENT_ID_TPC7_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) GOYA_ASYNC_EVENT_ID_MME_WACS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) GOYA_ASYNC_EVENT_ID_MME_WACSD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) GOYA_ASYNC_EVENT_ID_PSOC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) GOYA_ASYNC_EVENT_ID_TPC0_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) GOYA_ASYNC_EVENT_ID_TPC1_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) GOYA_ASYNC_EVENT_ID_TPC2_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) GOYA_ASYNC_EVENT_ID_TPC3_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) GOYA_ASYNC_EVENT_ID_TPC4_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) GOYA_ASYNC_EVENT_ID_TPC5_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) GOYA_ASYNC_EVENT_ID_TPC6_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) GOYA_ASYNC_EVENT_ID_TPC7_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) GOYA_ASYNC_EVENT_ID_TPC0_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) GOYA_ASYNC_EVENT_ID_TPC1_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) GOYA_ASYNC_EVENT_ID_TPC2_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) GOYA_ASYNC_EVENT_ID_TPC3_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) GOYA_ASYNC_EVENT_ID_TPC4_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) GOYA_ASYNC_EVENT_ID_TPC5_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) GOYA_ASYNC_EVENT_ID_TPC6_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) GOYA_ASYNC_EVENT_ID_TPC7_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) GOYA_ASYNC_EVENT_ID_MME_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) GOYA_ASYNC_EVENT_ID_MME_CMDQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) GOYA_ASYNC_EVENT_ID_DMA0_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) GOYA_ASYNC_EVENT_ID_DMA1_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) GOYA_ASYNC_EVENT_ID_DMA2_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) GOYA_ASYNC_EVENT_ID_DMA3_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) GOYA_ASYNC_EVENT_ID_DMA4_QM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) GOYA_ASYNC_EVENT_ID_DMA0_CH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) GOYA_ASYNC_EVENT_ID_DMA1_CH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) GOYA_ASYNC_EVENT_ID_DMA2_CH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) GOYA_ASYNC_EVENT_ID_DMA3_CH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) GOYA_ASYNC_EVENT_ID_DMA4_CH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) GOYA_ASYNC_EVENT_ID_DMA_BM_CH0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) GOYA_ASYNC_EVENT_ID_DMA_BM_CH1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) GOYA_ASYNC_EVENT_ID_DMA_BM_CH2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) GOYA_ASYNC_EVENT_ID_DMA_BM_CH3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) GOYA_ASYNC_EVENT_ID_DMA_BM_CH4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static int goya_mmu_clear_pgt_range(struct hl_device *hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) int goya_get_fixed_properties(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) prop->max_queues = GOYA_QUEUE_ID_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) prop->hw_queues_props = kcalloc(prop->max_queues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) sizeof(struct hw_queue_properties),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (!prop->hw_queues_props)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) prop->hw_queues_props[i].driver_only = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) prop->hw_queues_props[i].requires_kernel_cb = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) prop->hw_queues_props[i].driver_only = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) prop->hw_queues_props[i].requires_kernel_cb = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) NUMBER_OF_INT_HW_QUEUES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) prop->hw_queues_props[i].type = QUEUE_TYPE_INT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) prop->hw_queues_props[i].driver_only = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) prop->hw_queues_props[i].requires_kernel_cb = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) prop->dram_base_address = DRAM_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) prop->dram_size = DRAM_PHYS_DEFAULT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) prop->dram_end_address = prop->dram_base_address + prop->dram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) prop->dram_user_base_address = DRAM_BASE_ADDR_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) prop->sram_base_address = SRAM_BASE_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) prop->sram_size = SRAM_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) prop->sram_end_address = prop->sram_base_address + prop->sram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) prop->sram_user_base_address = prop->sram_base_address +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) SRAM_USER_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) prop->mmu_dram_default_page_addr = MMU_DRAM_DEFAULT_PAGE_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) prop->mmu_pgt_size = 0x800000; /* 8MB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) prop->mmu_pte_size = HL_PTE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) prop->mmu_hop_table_size = HOP_TABLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) prop->mmu_hop0_tables_total_size = HOP0_TABLES_TOTAL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) prop->dram_page_size = PAGE_SIZE_2MB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) prop->dmmu.hop0_shift = HOP0_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) prop->dmmu.hop1_shift = HOP1_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) prop->dmmu.hop2_shift = HOP2_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) prop->dmmu.hop3_shift = HOP3_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) prop->dmmu.hop4_shift = HOP4_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) prop->dmmu.hop0_mask = HOP0_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) prop->dmmu.hop1_mask = HOP1_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) prop->dmmu.hop2_mask = HOP2_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) prop->dmmu.hop3_mask = HOP3_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) prop->dmmu.hop4_mask = HOP4_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) prop->dmmu.start_addr = VA_DDR_SPACE_START;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) prop->dmmu.end_addr = VA_DDR_SPACE_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) prop->dmmu.page_size = PAGE_SIZE_2MB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) prop->dmmu.num_hops = MMU_ARCH_5_HOPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /* shifts and masks are the same in PMMU and DMMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) memcpy(&prop->pmmu, &prop->dmmu, sizeof(prop->dmmu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) prop->pmmu.start_addr = VA_HOST_SPACE_START;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) prop->pmmu.end_addr = VA_HOST_SPACE_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) prop->pmmu.page_size = PAGE_SIZE_4KB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) prop->pmmu.num_hops = MMU_ARCH_5_HOPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* PMMU and HPMMU are the same except of page size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) prop->pmmu_huge.page_size = PAGE_SIZE_2MB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) prop->dram_size_for_default_page_mapping = VA_DDR_SPACE_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) prop->cfg_size = CFG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) prop->max_asid = MAX_ASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) prop->num_of_events = GOYA_ASYNC_EVENT_ID_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) prop->high_pll = PLL_HIGH_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) prop->cb_pool_cb_cnt = GOYA_CB_POOL_CB_CNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) prop->cb_pool_cb_size = GOYA_CB_POOL_CB_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) prop->max_power_default = MAX_POWER_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) prop->tpc_enabled_mask = TPC_ENABLED_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) CARD_NAME_MAX_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) prop->max_pending_cs = GOYA_MAX_PENDING_CS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * goya_pci_bars_map - Map PCI BARS of Goya device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Request PCI regions and map them to kernel virtual addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static int goya_pci_bars_map(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static const char * const name[] = {"SRAM_CFG", "MSIX", "DDR"};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) bool is_wc[3] = {false, false, true};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) rc = hl_pci_bars_map(hdev, name, is_wc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) (CFG_BASE - SRAM_BASE_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) static u64 goya_set_ddr_bar_base(struct hl_device *hdev, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct hl_inbound_pci_region pci_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) u64 old_addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if ((goya) && (goya->ddr_bar_cur_addr == addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return old_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /* Inbound Region 1 - Bar 4 - Point to DDR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) pci_region.mode = PCI_BAR_MATCH_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) pci_region.bar = DDR_BAR_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) pci_region.addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) rc = hl_pci_set_inbound_region(hdev, 1, &pci_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return U64_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (goya) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) old_addr = goya->ddr_bar_cur_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) goya->ddr_bar_cur_addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return old_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * goya_init_iatu - Initialize the iATU unit inside the PCI controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * This is needed in case the firmware doesn't initialize the iATU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) static int goya_init_iatu(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct hl_inbound_pci_region inbound_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct hl_outbound_pci_region outbound_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) inbound_region.mode = PCI_BAR_MATCH_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) inbound_region.bar = SRAM_CFG_BAR_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) inbound_region.addr = SRAM_BASE_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* Inbound Region 1 - Bar 4 - Point to DDR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) inbound_region.mode = PCI_BAR_MATCH_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) inbound_region.bar = DDR_BAR_ID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) inbound_region.addr = DRAM_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) hdev->asic_funcs->set_dma_mask_from_fw(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) /* Outbound Region 0 - Point to Host */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) outbound_region.addr = HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) outbound_region.size = HOST_PHYS_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) rc = hl_pci_set_outbound_region(hdev, &outbound_region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * goya_early_init - GOYA early initialization code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * Verify PCI bars
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * Set DMA masks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * PCI controller initialization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * Map PCI bars
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static int goya_early_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct pci_dev *pdev = hdev->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) rc = goya_get_fixed_properties(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) dev_err(hdev->dev, "Failed to get fixed properties\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /* Check BAR sizes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (pci_resource_len(pdev, SRAM_CFG_BAR_ID) != CFG_BAR_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) SRAM_CFG_BAR_ID,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) (unsigned long long) pci_resource_len(pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) SRAM_CFG_BAR_ID),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) CFG_BAR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) rc = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) goto free_queue_props;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (pci_resource_len(pdev, MSIX_BAR_ID) != MSIX_BAR_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) "Not " HL_NAME "? BAR %d size %llu, expecting %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) MSIX_BAR_ID,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) (unsigned long long) pci_resource_len(pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) MSIX_BAR_ID),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) MSIX_BAR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) rc = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) goto free_queue_props;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) rc = hl_pci_init(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) mmCPU_BOOT_ERR0, GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) goto free_queue_props;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /* Goya Firmware does not support security */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) prop->fw_security_disabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) dev_info(hdev->dev, "firmware-level security is disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (!hdev->pldm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) val = RREG32(mmPSOC_GLOBAL_CONF_BOOT_STRAP_PINS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (val & PSOC_GLOBAL_CONF_BOOT_STRAP_PINS_SRIOV_EN_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) dev_warn(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) "PCI strap is not configured correctly, PCI bus errors may occur\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) free_queue_props:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) kfree(hdev->asic_prop.hw_queues_props);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * goya_early_fini - GOYA early finalization code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * Unmap PCI bars
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) static int goya_early_fini(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) kfree(hdev->asic_prop.hw_queues_props);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) hl_pci_fini(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) static void goya_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /* mask to zero the MMBP and ASID bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) WREG32_AND(reg, ~0x7FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) WREG32_OR(reg, asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) static void goya_qman0_set_security(struct hl_device *hdev, bool secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_FULLY_TRUSTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) WREG32(mmDMA_QM_0_GLBL_PROT, QMAN_DMA_PARTLY_TRUSTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) RREG32(mmDMA_QM_0_GLBL_PROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * goya_fetch_psoc_frequency - Fetch PSOC frequency values
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) static void goya_fetch_psoc_frequency(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) u32 trace_freq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) u32 pll_clk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) u32 div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) u32 div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) u32 nr = RREG32(mmPSOC_PCI_PLL_NR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) u32 nf = RREG32(mmPSOC_PCI_PLL_NF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) u32 od = RREG32(mmPSOC_PCI_PLL_OD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (div_sel == DIV_SEL_REF_CLK || div_sel == DIV_SEL_DIVIDED_REF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (div_sel == DIV_SEL_REF_CLK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) trace_freq = PLL_REF_CLK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) trace_freq = PLL_REF_CLK / (div_fctr + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) } else if (div_sel == DIV_SEL_PLL_CLK ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) div_sel == DIV_SEL_DIVIDED_PLL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) pll_clk = PLL_REF_CLK * (nf + 1) / ((nr + 1) * (od + 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) if (div_sel == DIV_SEL_PLL_CLK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) trace_freq = pll_clk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) trace_freq = pll_clk / (div_fctr + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) dev_warn(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) "Received invalid div select value: %d", div_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) prop->psoc_timestamp_frequency = trace_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) prop->psoc_pci_pll_nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) prop->psoc_pci_pll_nf = nf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) prop->psoc_pci_pll_od = od;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) prop->psoc_pci_pll_div_factor = div_fctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) int goya_late_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) goya_fetch_psoc_frequency(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) rc = goya_mmu_clear_pgt_range(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) "Failed to clear MMU page tables range %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) rc = goya_mmu_set_dram_default_page(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) dev_err(hdev->dev, "Failed to set DRAM default page %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) rc = goya_mmu_add_mappings_for_device_cpu(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) rc = goya_init_cpu_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) rc = goya_test_cpu_queue(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) rc = goya_cpucp_info_get(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) dev_err(hdev->dev, "Failed to get cpucp info %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /* Now that we have the DRAM size in ASIC prop, we need to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * its size and configure the DMA_IF DDR wrap protection (which is in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * the MMU block) accordingly. The value is the log2 of the DRAM size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) WREG32(mmMMU_LOG2_DDR_SIZE, ilog2(prop->dram_size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) "Failed to enable PCI access from CPU %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) GOYA_ASYNC_EVENT_ID_INTS_REGISTER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * goya_late_fini - GOYA late tear-down code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * Free sensors allocated structures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) void goya_late_fini(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) const struct hwmon_channel_info **channel_info_arr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (!hdev->hl_chip_info->info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) channel_info_arr = hdev->hl_chip_info->info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) while (channel_info_arr[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) kfree(channel_info_arr[i]->config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) kfree(channel_info_arr[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) kfree(channel_info_arr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) hdev->hl_chip_info->info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * goya_sw_init - Goya software initialization code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) static int goya_sw_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) struct goya_device *goya;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /* Allocate device structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) goya = kzalloc(sizeof(*goya), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (!goya)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) /* according to goya_init_iatu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) goya->ddr_bar_cur_addr = DRAM_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goya->mme_clk = GOYA_PLL_FREQ_LOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) goya->tpc_clk = GOYA_PLL_FREQ_LOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) goya->ic_clk = GOYA_PLL_FREQ_LOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) hdev->asic_specific = goya;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /* Create DMA pool for small allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) &hdev->pdev->dev, GOYA_DMA_POOL_BLK_SIZE, 8, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (!hdev->dma_pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) dev_err(hdev->dev, "failed to create DMA pool\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) goto free_goya_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) hdev->cpu_accessible_dma_mem =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) hdev->asic_funcs->asic_dma_alloc_coherent(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) HL_CPU_ACCESSIBLE_MEM_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) &hdev->cpu_accessible_dma_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) GFP_KERNEL | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (!hdev->cpu_accessible_dma_mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) goto free_dma_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) dev_dbg(hdev->dev, "cpu accessible memory at bus address %pad\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) &hdev->cpu_accessible_dma_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (!hdev->cpu_accessible_dma_pool) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) "Failed to create CPU accessible DMA pool\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) goto free_cpu_dma_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) rc = gen_pool_add(hdev->cpu_accessible_dma_pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) (uintptr_t) hdev->cpu_accessible_dma_mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) HL_CPU_ACCESSIBLE_MEM_SIZE, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) "Failed to add memory to CPU accessible DMA pool\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) goto free_cpu_accessible_dma_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) spin_lock_init(&goya->hw_queues_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) hdev->supports_coresight = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) hdev->supports_soft_reset = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) free_cpu_accessible_dma_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) gen_pool_destroy(hdev->cpu_accessible_dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) free_cpu_dma_mem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) hdev->asic_funcs->asic_dma_free_coherent(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) HL_CPU_ACCESSIBLE_MEM_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) hdev->cpu_accessible_dma_mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) hdev->cpu_accessible_dma_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) free_dma_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) dma_pool_destroy(hdev->dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) free_goya_device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) kfree(goya);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * goya_sw_fini - Goya software tear-down code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) static int goya_sw_fini(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) gen_pool_destroy(hdev->cpu_accessible_dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) hdev->asic_funcs->asic_dma_free_coherent(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) HL_CPU_ACCESSIBLE_MEM_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) hdev->cpu_accessible_dma_mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) hdev->cpu_accessible_dma_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) dma_pool_destroy(hdev->dma_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) kfree(goya);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) static void goya_init_dma_qman(struct hl_device *hdev, int dma_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) dma_addr_t bus_address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) u32 mtr_base_lo, mtr_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) u32 reg_off = dma_id * (mmDMA_QM_1_PQ_PI - mmDMA_QM_0_PQ_PI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) u32 dma_err_cfg = QMAN_DMA_ERR_MSG_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) WREG32(mmDMA_QM_0_PQ_BASE_LO + reg_off, lower_32_bits(bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) WREG32(mmDMA_QM_0_PQ_BASE_HI + reg_off, upper_32_bits(bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) WREG32(mmDMA_QM_0_PQ_SIZE + reg_off, ilog2(HL_QUEUE_LENGTH));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) WREG32(mmDMA_QM_0_PQ_PI + reg_off, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) WREG32(mmDMA_QM_0_PQ_CI + reg_off, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) WREG32(mmDMA_QM_0_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) WREG32(mmDMA_QM_0_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) WREG32(mmDMA_QM_0_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) WREG32(mmDMA_QM_0_GLBL_ERR_WDATA + reg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) GOYA_ASYNC_EVENT_ID_DMA0_QM + dma_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) /* PQ has buffer of 2 cache lines, while CQ has 8 lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) WREG32(mmDMA_QM_0_PQ_CFG1 + reg_off, 0x00020002);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) WREG32(mmDMA_QM_0_CQ_CFG1 + reg_off, 0x00080008);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (goya->hw_cap_initialized & HW_CAP_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_PARTLY_TRUSTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) WREG32(mmDMA_QM_0_GLBL_PROT + reg_off, QMAN_DMA_FULLY_TRUSTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (hdev->stop_on_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) dma_err_cfg |= 1 << DMA_QM_0_GLBL_ERR_CFG_DMA_STOP_ON_ERR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) WREG32(mmDMA_QM_0_GLBL_ERR_CFG + reg_off, dma_err_cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) WREG32(mmDMA_QM_0_GLBL_CFG0 + reg_off, QMAN_DMA_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) static void goya_init_dma_ch(struct hl_device *hdev, int dma_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) u64 sob_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) u32 reg_off = dma_id * (mmDMA_CH_1_CFG1 - mmDMA_CH_0_CFG1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) WREG32(mmDMA_CH_0_ERRMSG_ADDR_LO + reg_off, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) WREG32(mmDMA_CH_0_ERRMSG_ADDR_HI + reg_off, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) WREG32(mmDMA_CH_0_ERRMSG_WDATA + reg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) GOYA_ASYNC_EVENT_ID_DMA0_CH + dma_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (dma_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) (dma_id - 1) * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) WREG32(mmDMA_CH_0_WR_COMP_ADDR_HI + reg_off, upper_32_bits(sob_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) WREG32(mmDMA_CH_0_WR_COMP_WDATA + reg_off, 0x80000001);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * goya_init_dma_qmans - Initialize QMAN DMA registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * Initialize the H/W registers of the QMAN DMA channels
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) void goya_init_dma_qmans(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) struct hl_hw_queue *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (goya->hw_cap_initialized & HW_CAP_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) q = &hdev->kernel_queues[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++, q++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) q->cq_id = q->msi_vec = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) goya_init_dma_qman(hdev, i, q->bus_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) goya_init_dma_ch(hdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) goya->hw_cap_initialized |= HW_CAP_DMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) * goya_disable_external_queues - Disable external queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) static void goya_disable_external_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (!(goya->hw_cap_initialized & HW_CAP_DMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) WREG32(mmDMA_QM_0_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) WREG32(mmDMA_QM_1_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) WREG32(mmDMA_QM_2_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) WREG32(mmDMA_QM_3_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) WREG32(mmDMA_QM_4_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) static int goya_stop_queue(struct hl_device *hdev, u32 cfg_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) u32 cp_sts_reg, u32 glbl_sts0_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) /* use the values of TPC0 as they are all the same*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) WREG32(cfg_reg, 1 << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) status = RREG32(cp_sts_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) if (status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) rc = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) cp_sts_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) !(status & TPC0_QM_CP_STS_FENCE_IN_PROGRESS_MASK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) QMAN_FENCE_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) /* if QMAN is stuck in fence no need to check for stop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) rc = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) glbl_sts0_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) (status & TPC0_QM_GLBL_STS0_CP_IS_STOP_MASK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) QMAN_STOP_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) "Timeout while waiting for QMAN to stop\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) * goya_stop_external_queues - Stop external queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) static int goya_stop_external_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) int rc, retval = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (!(goya->hw_cap_initialized & HW_CAP_DMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) mmDMA_QM_0_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) mmDMA_QM_0_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) mmDMA_QM_0_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) dev_err(hdev->dev, "failed to stop DMA QMAN 0\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) mmDMA_QM_1_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) mmDMA_QM_1_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) mmDMA_QM_1_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) dev_err(hdev->dev, "failed to stop DMA QMAN 1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) mmDMA_QM_2_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) mmDMA_QM_2_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) mmDMA_QM_2_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) dev_err(hdev->dev, "failed to stop DMA QMAN 2\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) mmDMA_QM_3_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) mmDMA_QM_3_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) mmDMA_QM_3_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) dev_err(hdev->dev, "failed to stop DMA QMAN 3\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) mmDMA_QM_4_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) mmDMA_QM_4_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) mmDMA_QM_4_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) dev_err(hdev->dev, "failed to stop DMA QMAN 4\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * goya_init_cpu_queues - Initialize PQ/CQ/EQ of CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) int goya_init_cpu_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) struct hl_eq *eq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GOYA_QUEUE_ID_CPU_PQ];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (!hdev->cpu_queues_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) if (goya->hw_cap_initialized & HW_CAP_CPU_Q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) eq = &hdev->event_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) WREG32(mmCPU_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) WREG32(mmCPU_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) WREG32(mmCPU_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) WREG32(mmCPU_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) WREG32(mmCPU_CQ_BASE_ADDR_LOW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) lower_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) WREG32(mmCPU_CQ_BASE_ADDR_HIGH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) upper_32_bits(VA_CPU_ACCESSIBLE_MEM_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) WREG32(mmCPU_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) WREG32(mmCPU_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) WREG32(mmCPU_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /* Used for EQ CI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) WREG32(mmCPU_EQ_CI, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) WREG32(mmCPU_IF_PF_PQ_PI, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) WREG32(mmCPU_PQ_INIT_STATUS, PQ_INIT_STATUS_READY_FOR_CP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) GOYA_ASYNC_EVENT_ID_PI_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) err = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) mmCPU_PQ_INIT_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) (status == PQ_INIT_STATUS_READY_FOR_HOST),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) GOYA_CPU_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) "Failed to setup communication with device CPU\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) goya->hw_cap_initialized |= HW_CAP_CPU_Q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) static void goya_set_pll_refclk(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) WREG32(mmCPU_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) WREG32(mmCPU_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) WREG32(mmCPU_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) WREG32(mmCPU_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) WREG32(mmIC_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) WREG32(mmIC_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) WREG32(mmIC_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) WREG32(mmIC_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) WREG32(mmMC_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) WREG32(mmMC_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) WREG32(mmMC_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) WREG32(mmMC_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) WREG32(mmPSOC_MME_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) WREG32(mmPSOC_MME_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) WREG32(mmPSOC_MME_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) WREG32(mmPSOC_MME_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) WREG32(mmPSOC_PCI_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) WREG32(mmPSOC_PCI_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) WREG32(mmPSOC_PCI_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) WREG32(mmPSOC_PCI_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) WREG32(mmPSOC_EMMC_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) WREG32(mmPSOC_EMMC_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) WREG32(mmPSOC_EMMC_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) WREG32(mmPSOC_EMMC_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) WREG32(mmTPC_PLL_DIV_SEL_0, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) WREG32(mmTPC_PLL_DIV_SEL_1, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) WREG32(mmTPC_PLL_DIV_SEL_2, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) WREG32(mmTPC_PLL_DIV_SEL_3, 0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) static void goya_disable_clk_rlx(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) WREG32(mmPSOC_MME_PLL_CLK_RLX_0, 0x100010);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) WREG32(mmIC_PLL_CLK_RLX_0, 0x100010);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static void _goya_tpc_mbist_workaround(struct hl_device *hdev, u8 tpc_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) u64 tpc_eml_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) u32 val, tpc_offset, tpc_eml_offset, tpc_slm_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) int err, slm_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) tpc_offset = tpc_id * 0x40000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) tpc_eml_offset = tpc_id * 0x200000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) tpc_eml_address = (mmTPC0_EML_CFG_BASE + tpc_eml_offset - CFG_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) tpc_slm_offset = tpc_eml_address + 0x100000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * Workaround for Bug H2 #2443 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * "TPC SB is not initialized on chip reset"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) val = RREG32(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) if (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_ACTIVE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) dev_warn(hdev->dev, "TPC%d MBIST ACTIVE is not cleared\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) tpc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) WREG32(mmTPC0_CFG_FUNC_MBIST_PAT + tpc_offset, val & 0xFFFFF000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_0 + tpc_offset, 0x37FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_1 + tpc_offset, 0x303F);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_2 + tpc_offset, 0x71FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_3 + tpc_offset, 0x71FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_4 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_5 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_6 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_7 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_8 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) WREG32(mmTPC0_CFG_FUNC_MBIST_MEM_9 + tpc_offset, 0x70FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) WREG32_OR(mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) 1 << TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_START_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) err = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) mmTPC0_CFG_FUNC_MBIST_CNTRL + tpc_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) (val & TPC0_CFG_FUNC_MBIST_CNTRL_MBIST_DONE_MASK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) HL_DEVICE_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) "Timeout while waiting for TPC%d MBIST DONE\n", tpc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) WREG32_OR(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) 1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) msleep(GOYA_RESET_WAIT_MSEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) WREG32_AND(mmTPC0_EML_CFG_DBG_CNT + tpc_eml_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) ~(1 << TPC0_EML_CFG_DBG_CNT_CORE_RST_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) msleep(GOYA_RESET_WAIT_MSEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) for (slm_index = 0 ; slm_index < 256 ; slm_index++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) WREG32(tpc_slm_offset + (slm_index << 2), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) val = RREG32(tpc_slm_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) static void goya_tpc_mbist_workaround(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) if (goya->hw_cap_initialized & HW_CAP_TPC_MBIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) /* Workaround for H2 #2443 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) for (i = 0 ; i < TPC_MAX_NUM ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) _goya_tpc_mbist_workaround(hdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) goya->hw_cap_initialized |= HW_CAP_TPC_MBIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) * goya_init_golden_registers - Initialize golden registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * Initialize the H/W registers of the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) static void goya_init_golden_registers(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) u32 polynom[10], tpc_intr_mask, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) if (goya->hw_cap_initialized & HW_CAP_GOLDEN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) polynom[0] = 0x00020080;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) polynom[1] = 0x00401000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) polynom[2] = 0x00200800;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) polynom[3] = 0x00002000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) polynom[4] = 0x00080200;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) polynom[5] = 0x00040100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) polynom[6] = 0x00100400;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) polynom[7] = 0x00004000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) polynom[8] = 0x00010000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) polynom[9] = 0x00008000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) /* Mask all arithmetic interrupts from TPC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) tpc_intr_mask = 0x7FFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x20000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) WREG32(mmSRAM_Y0_X0_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) WREG32(mmSRAM_Y0_X1_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) WREG32(mmSRAM_Y0_X2_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) WREG32(mmSRAM_Y0_X3_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) WREG32(mmSRAM_Y0_X4_RTR_HBW_RD_RQ_L_ARB + offset, 0x302);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_L_ARB + offset, 0x204);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_L_ARB + offset, 0x204);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_L_ARB + offset, 0x204);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_L_ARB + offset, 0x204);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_L_ARB + offset, 0x204);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_E_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_E_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_E_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_E_ARB + offset, 0x207);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_E_ARB + offset, 0x207);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) WREG32(mmSRAM_Y0_X0_RTR_HBW_DATA_W_ARB + offset, 0x207);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) WREG32(mmSRAM_Y0_X1_RTR_HBW_DATA_W_ARB + offset, 0x207);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) WREG32(mmSRAM_Y0_X2_RTR_HBW_DATA_W_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) WREG32(mmSRAM_Y0_X3_RTR_HBW_DATA_W_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) WREG32(mmSRAM_Y0_X4_RTR_HBW_DATA_W_ARB + offset, 0x206);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_E_ARB + offset, 0x101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_E_ARB + offset, 0x102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_E_ARB + offset, 0x103);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_E_ARB + offset, 0x104);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_E_ARB + offset, 0x105);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) WREG32(mmSRAM_Y0_X0_RTR_HBW_WR_RS_W_ARB + offset, 0x105);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) WREG32(mmSRAM_Y0_X1_RTR_HBW_WR_RS_W_ARB + offset, 0x104);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) WREG32(mmSRAM_Y0_X2_RTR_HBW_WR_RS_W_ARB + offset, 0x103);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) WREG32(mmSRAM_Y0_X3_RTR_HBW_WR_RS_W_ARB + offset, 0x102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) WREG32(mmSRAM_Y0_X4_RTR_HBW_WR_RS_W_ARB + offset, 0x101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) WREG32(mmMME_STORE_MAX_CREDIT, 0x21);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) WREG32(mmMME_AGU, 0x0f0f0f10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) WREG32(mmMME_SEI_MASK, ~0x0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) WREG32(mmMME6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) WREG32(mmMME5_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) WREG32(mmMME4_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) WREG32(mmMME3_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) WREG32(mmMME2_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) WREG32(mmMME1_RTR_HBW_RD_RQ_N_ARB, 0x07010701);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) WREG32(mmMME6_RTR_HBW_RD_RQ_S_ARB, 0x04010401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) WREG32(mmMME5_RTR_HBW_RD_RQ_S_ARB, 0x04050401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) WREG32(mmMME4_RTR_HBW_RD_RQ_S_ARB, 0x03070301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) WREG32(mmMME3_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) WREG32(mmMME2_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) WREG32(mmMME1_RTR_HBW_RD_RQ_S_ARB, 0x01050105);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) WREG32(mmMME6_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) WREG32(mmMME5_RTR_HBW_RD_RQ_W_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) WREG32(mmMME4_RTR_HBW_RD_RQ_W_ARB, 0x01040301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) WREG32(mmMME3_RTR_HBW_RD_RQ_W_ARB, 0x01030401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) WREG32(mmMME2_RTR_HBW_RD_RQ_W_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) WREG32(mmMME1_RTR_HBW_RD_RQ_W_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) WREG32(mmMME6_RTR_HBW_WR_RQ_N_ARB, 0x02020202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) WREG32(mmMME5_RTR_HBW_WR_RQ_N_ARB, 0x01070101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) WREG32(mmMME4_RTR_HBW_WR_RQ_N_ARB, 0x02020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) WREG32(mmMME3_RTR_HBW_WR_RQ_N_ARB, 0x07020701);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) WREG32(mmMME2_RTR_HBW_WR_RQ_N_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) WREG32(mmMME6_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) WREG32(mmMME5_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) WREG32(mmMME4_RTR_HBW_WR_RQ_S_ARB, 0x07020701);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) WREG32(mmMME3_RTR_HBW_WR_RQ_S_ARB, 0x02020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) WREG32(mmMME2_RTR_HBW_WR_RQ_S_ARB, 0x01070101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) WREG32(mmMME1_RTR_HBW_WR_RQ_S_ARB, 0x01020102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) WREG32(mmMME6_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) WREG32(mmMME5_RTR_HBW_WR_RQ_W_ARB, 0x01020701);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) WREG32(mmMME4_RTR_HBW_WR_RQ_W_ARB, 0x07020707);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) WREG32(mmMME3_RTR_HBW_WR_RQ_W_ARB, 0x01020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) WREG32(mmMME2_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) WREG32(mmMME1_RTR_HBW_WR_RQ_W_ARB, 0x01070201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) WREG32(mmMME6_RTR_HBW_RD_RS_N_ARB, 0x01070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) WREG32(mmMME5_RTR_HBW_RD_RS_N_ARB, 0x01070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) WREG32(mmMME4_RTR_HBW_RD_RS_N_ARB, 0x01060102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) WREG32(mmMME3_RTR_HBW_RD_RS_N_ARB, 0x01040102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) WREG32(mmMME2_RTR_HBW_RD_RS_N_ARB, 0x01020102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) WREG32(mmMME1_RTR_HBW_RD_RS_N_ARB, 0x01020107);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) WREG32(mmMME6_RTR_HBW_RD_RS_S_ARB, 0x01020106);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) WREG32(mmMME5_RTR_HBW_RD_RS_S_ARB, 0x01020102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) WREG32(mmMME4_RTR_HBW_RD_RS_S_ARB, 0x01040102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) WREG32(mmMME3_RTR_HBW_RD_RS_S_ARB, 0x01060102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) WREG32(mmMME2_RTR_HBW_RD_RS_S_ARB, 0x01070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) WREG32(mmMME1_RTR_HBW_RD_RS_S_ARB, 0x01070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) WREG32(mmMME6_RTR_HBW_RD_RS_E_ARB, 0x01020702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) WREG32(mmMME5_RTR_HBW_RD_RS_E_ARB, 0x01020702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) WREG32(mmMME4_RTR_HBW_RD_RS_E_ARB, 0x01040602);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) WREG32(mmMME3_RTR_HBW_RD_RS_E_ARB, 0x01060402);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) WREG32(mmMME2_RTR_HBW_RD_RS_E_ARB, 0x01070202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) WREG32(mmMME1_RTR_HBW_RD_RS_E_ARB, 0x01070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) WREG32(mmMME6_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) WREG32(mmMME5_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) WREG32(mmMME4_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) WREG32(mmMME3_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) WREG32(mmMME2_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) WREG32(mmMME1_RTR_HBW_RD_RS_W_ARB, 0x01060401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) WREG32(mmMME6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) WREG32(mmMME5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) WREG32(mmMME4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) WREG32(mmMME3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) WREG32(mmMME2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) WREG32(mmMME1_RTR_HBW_WR_RS_N_ARB, 0x01010107);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) WREG32(mmMME6_RTR_HBW_WR_RS_S_ARB, 0x01010107);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) WREG32(mmMME5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) WREG32(mmMME4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) WREG32(mmMME3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) WREG32(mmMME2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) WREG32(mmMME1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) WREG32(mmMME6_RTR_HBW_WR_RS_E_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) WREG32(mmMME5_RTR_HBW_WR_RS_E_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) WREG32(mmMME4_RTR_HBW_WR_RS_E_ARB, 0x01040301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) WREG32(mmMME3_RTR_HBW_WR_RS_E_ARB, 0x01030401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) WREG32(mmMME2_RTR_HBW_WR_RS_E_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) WREG32(mmMME1_RTR_HBW_WR_RS_E_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) WREG32(mmMME6_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) WREG32(mmMME5_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) WREG32(mmMME4_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) WREG32(mmMME3_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) WREG32(mmMME2_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) WREG32(mmMME1_RTR_HBW_WR_RS_W_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) WREG32(mmTPC1_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) WREG32(mmTPC1_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) WREG32(mmTPC1_RTR_HBW_RD_RQ_E_ARB, 0x01060101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) WREG32(mmTPC1_RTR_HBW_WR_RQ_N_ARB, 0x02020102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) WREG32(mmTPC1_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) WREG32(mmTPC1_RTR_HBW_WR_RQ_E_ARB, 0x02070202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) WREG32(mmTPC1_RTR_HBW_RD_RS_N_ARB, 0x01020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) WREG32(mmTPC1_RTR_HBW_RD_RS_S_ARB, 0x01070201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) WREG32(mmTPC1_RTR_HBW_RD_RS_W_ARB, 0x01070202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) WREG32(mmTPC1_RTR_HBW_WR_RS_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) WREG32(mmTPC1_RTR_HBW_WR_RS_S_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) WREG32(mmTPC1_RTR_HBW_WR_RS_W_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) WREG32(mmTPC2_RTR_HBW_RD_RQ_N_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) WREG32(mmTPC2_RTR_HBW_RD_RQ_S_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) WREG32(mmTPC2_RTR_HBW_RD_RQ_E_ARB, 0x01010201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) WREG32(mmTPC2_RTR_HBW_WR_RQ_N_ARB, 0x02040102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) WREG32(mmTPC2_RTR_HBW_WR_RQ_S_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) WREG32(mmTPC2_RTR_HBW_WR_RQ_E_ARB, 0x02060202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) WREG32(mmTPC2_RTR_HBW_RD_RS_N_ARB, 0x01020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) WREG32(mmTPC2_RTR_HBW_RD_RS_S_ARB, 0x01070201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) WREG32(mmTPC2_RTR_HBW_RD_RS_W_ARB, 0x01070202);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) WREG32(mmTPC2_RTR_HBW_WR_RS_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) WREG32(mmTPC2_RTR_HBW_WR_RS_S_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) WREG32(mmTPC2_RTR_HBW_WR_RS_W_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) WREG32(mmTPC3_RTR_HBW_RD_RQ_N_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) WREG32(mmTPC3_RTR_HBW_RD_RQ_S_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) WREG32(mmTPC3_RTR_HBW_RD_RQ_E_ARB, 0x01040301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) WREG32(mmTPC3_RTR_HBW_WR_RQ_N_ARB, 0x02060102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) WREG32(mmTPC3_RTR_HBW_WR_RQ_S_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) WREG32(mmTPC3_RTR_HBW_WR_RQ_E_ARB, 0x01040301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) WREG32(mmTPC3_RTR_HBW_RD_RS_N_ARB, 0x01040201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) WREG32(mmTPC3_RTR_HBW_RD_RS_S_ARB, 0x01060201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) WREG32(mmTPC3_RTR_HBW_RD_RS_W_ARB, 0x01060402);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) WREG32(mmTPC3_RTR_HBW_WR_RS_N_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) WREG32(mmTPC3_RTR_HBW_WR_RS_S_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) WREG32(mmTPC3_RTR_HBW_WR_RS_W_ARB, 0x01030401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) WREG32(mmTPC4_RTR_HBW_RD_RQ_N_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) WREG32(mmTPC4_RTR_HBW_RD_RQ_S_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) WREG32(mmTPC4_RTR_HBW_RD_RQ_E_ARB, 0x01030401);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) WREG32(mmTPC4_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) WREG32(mmTPC4_RTR_HBW_WR_RQ_S_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) WREG32(mmTPC4_RTR_HBW_WR_RQ_E_ARB, 0x02060702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) WREG32(mmTPC4_RTR_HBW_RD_RS_N_ARB, 0x01060201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) WREG32(mmTPC4_RTR_HBW_RD_RS_S_ARB, 0x01040201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) WREG32(mmTPC4_RTR_HBW_RD_RS_W_ARB, 0x01040602);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) WREG32(mmTPC4_RTR_HBW_WR_RS_N_ARB, 0x01030101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) WREG32(mmTPC4_RTR_HBW_WR_RS_S_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) WREG32(mmTPC4_RTR_HBW_WR_RS_W_ARB, 0x01040301);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) WREG32(mmTPC5_RTR_HBW_RD_RQ_N_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) WREG32(mmTPC5_RTR_HBW_RD_RQ_S_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) WREG32(mmTPC5_RTR_HBW_RD_RQ_E_ARB, 0x01200501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) WREG32(mmTPC5_RTR_HBW_WR_RQ_N_ARB, 0x02070102);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) WREG32(mmTPC5_RTR_HBW_WR_RQ_S_ARB, 0x01020101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) WREG32(mmTPC5_RTR_HBW_WR_RQ_E_ARB, 0x02020602);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) WREG32(mmTPC5_RTR_HBW_RD_RS_N_ARB, 0x01070201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) WREG32(mmTPC5_RTR_HBW_RD_RS_S_ARB, 0x01020201);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) WREG32(mmTPC5_RTR_HBW_RD_RS_W_ARB, 0x01020702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) WREG32(mmTPC5_RTR_HBW_WR_RS_N_ARB, 0x01040101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) WREG32(mmTPC5_RTR_HBW_WR_RS_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) WREG32(mmTPC5_RTR_HBW_WR_RS_W_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) WREG32(mmTPC6_RTR_HBW_RD_RQ_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) WREG32(mmTPC6_RTR_HBW_RD_RQ_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) WREG32(mmTPC6_RTR_HBW_RD_RQ_E_ARB, 0x01010601);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) WREG32(mmTPC6_RTR_HBW_WR_RQ_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) WREG32(mmTPC6_RTR_HBW_WR_RQ_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) WREG32(mmTPC6_RTR_HBW_WR_RQ_E_ARB, 0x02020702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) WREG32(mmTPC6_RTR_HBW_RD_RS_N_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) WREG32(mmTPC6_RTR_HBW_RD_RS_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) WREG32(mmTPC6_RTR_HBW_RD_RS_W_ARB, 0x01020702);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) WREG32(mmTPC6_RTR_HBW_WR_RS_N_ARB, 0x01050101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) WREG32(mmTPC6_RTR_HBW_WR_RS_S_ARB, 0x01010101);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) WREG32(mmTPC6_RTR_HBW_WR_RS_W_ARB, 0x01010501);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) for (i = 0, offset = 0 ; i < 10 ; i++, offset += 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) WREG32(mmMME1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) WREG32(mmMME2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) WREG32(mmMME3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) WREG32(mmMME4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) WREG32(mmMME5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) WREG32(mmMME6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) WREG32(mmTPC0_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) WREG32(mmTPC1_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) WREG32(mmTPC2_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) WREG32(mmTPC3_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) WREG32(mmTPC4_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) WREG32(mmTPC5_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) WREG32(mmTPC6_RTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) WREG32(mmTPC7_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) WREG32(mmPCI_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) WREG32(mmDMA_NRTR_SPLIT_COEF_0 + offset, polynom[i] >> 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) for (i = 0, offset = 0 ; i < 6 ; i++, offset += 0x40000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) WREG32(mmMME1_RTR_SCRAMB_EN + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) 1 << MME1_RTR_SCRAMB_EN_VAL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) WREG32(mmMME1_RTR_NON_LIN_SCRAMB + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) 1 << MME1_RTR_NON_LIN_SCRAMB_EN_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) for (i = 0, offset = 0 ; i < 8 ; i++, offset += 0x40000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) * Workaround for Bug H2 #2441 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * "ST.NOP set trace event illegal opcode"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) WREG32(mmTPC0_CFG_TPC_INTR_MASK + offset, tpc_intr_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) WREG32(mmTPC0_NRTR_SCRAMB_EN + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) 1 << TPC0_NRTR_SCRAMB_EN_VAL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) WREG32(mmTPC0_NRTR_NON_LIN_SCRAMB + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) 1 << TPC0_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) WREG32_FIELD(TPC0_CFG_MSS_CONFIG, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) ICACHE_FETCH_LINE_NUM, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) WREG32(mmDMA_NRTR_SCRAMB_EN, 1 << DMA_NRTR_SCRAMB_EN_VAL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) WREG32(mmDMA_NRTR_NON_LIN_SCRAMB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) 1 << DMA_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) WREG32(mmPCI_NRTR_SCRAMB_EN, 1 << PCI_NRTR_SCRAMB_EN_VAL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) WREG32(mmPCI_NRTR_NON_LIN_SCRAMB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) 1 << PCI_NRTR_NON_LIN_SCRAMB_EN_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) * Workaround for H2 #HW-23 bug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) * Set DMA max outstanding read requests to 240 on DMA CH 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) * This limitation is still large enough to not affect Gen4 bandwidth.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * We need to only limit that DMA channel because the user can only read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) * from Host using DMA CH 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) WREG32(mmDMA_CH_1_CFG0, 0x0fff00F0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) goya->hw_cap_initialized |= HW_CAP_GOLDEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) static void goya_init_mme_qman(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) u32 mtr_base_lo, mtr_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) u64 qman_base_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) qman_base_addr = hdev->asic_prop.sram_base_address +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) MME_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) WREG32(mmMME_QM_PQ_BASE_LO, lower_32_bits(qman_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) WREG32(mmMME_QM_PQ_BASE_HI, upper_32_bits(qman_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) WREG32(mmMME_QM_PQ_SIZE, ilog2(MME_QMAN_LENGTH));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) WREG32(mmMME_QM_PQ_PI, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) WREG32(mmMME_QM_PQ_CI, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) WREG32(mmMME_QM_CP_LDMA_SRC_BASE_LO_OFFSET, 0x10C0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) WREG32(mmMME_QM_CP_LDMA_SRC_BASE_HI_OFFSET, 0x10C4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) WREG32(mmMME_QM_CP_LDMA_TSIZE_OFFSET, 0x10C8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) WREG32(mmMME_QM_CP_LDMA_COMMIT_OFFSET, 0x10CC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) WREG32(mmMME_QM_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_LO, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) WREG32(mmMME_QM_CP_MSG_BASE1_ADDR_HI, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) /* QMAN CQ has 8 cache lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) WREG32(mmMME_QM_CQ_CFG1, 0x00080008);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) WREG32(mmMME_QM_GLBL_ERR_ADDR_LO, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) WREG32(mmMME_QM_GLBL_ERR_ADDR_HI, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) WREG32(mmMME_QM_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_QM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) WREG32(mmMME_QM_GLBL_ERR_CFG, QMAN_MME_ERR_MSG_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) WREG32(mmMME_QM_GLBL_PROT, QMAN_MME_ERR_PROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) WREG32(mmMME_QM_GLBL_CFG0, QMAN_MME_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) static void goya_init_mme_cmdq(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) u32 mtr_base_lo, mtr_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_LO, mtr_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) WREG32(mmMME_CMDQ_CP_MSG_BASE0_ADDR_HI, mtr_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_LO, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) WREG32(mmMME_CMDQ_CP_MSG_BASE1_ADDR_HI, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) /* CMDQ CQ has 20 cache lines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) WREG32(mmMME_CMDQ_CQ_CFG1, 0x00140014);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_LO, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) WREG32(mmMME_CMDQ_GLBL_ERR_ADDR_HI, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) WREG32(mmMME_CMDQ_GLBL_ERR_WDATA, GOYA_ASYNC_EVENT_ID_MME_CMDQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) WREG32(mmMME_CMDQ_GLBL_ERR_CFG, CMDQ_MME_ERR_MSG_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) WREG32(mmMME_CMDQ_GLBL_PROT, CMDQ_MME_ERR_PROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) WREG32(mmMME_CMDQ_GLBL_CFG0, CMDQ_MME_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) void goya_init_mme_qmans(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) if (goya->hw_cap_initialized & HW_CAP_MME)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) WREG32(mmMME_SM_BASE_ADDRESS_LOW, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) WREG32(mmMME_SM_BASE_ADDRESS_HIGH, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) goya_init_mme_qman(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) goya_init_mme_cmdq(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) goya->hw_cap_initialized |= HW_CAP_MME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) static void goya_init_tpc_qman(struct hl_device *hdev, u32 base_off, int tpc_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) u32 mtr_base_lo, mtr_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) u64 qman_base_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) u32 reg_off = tpc_id * (mmTPC1_QM_PQ_PI - mmTPC0_QM_PQ_PI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) qman_base_addr = hdev->asic_prop.sram_base_address + base_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) WREG32(mmTPC0_QM_PQ_BASE_LO + reg_off, lower_32_bits(qman_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) WREG32(mmTPC0_QM_PQ_BASE_HI + reg_off, upper_32_bits(qman_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) WREG32(mmTPC0_QM_PQ_SIZE + reg_off, ilog2(TPC_QMAN_LENGTH));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) WREG32(mmTPC0_QM_PQ_PI + reg_off, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) WREG32(mmTPC0_QM_PQ_CI + reg_off, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET + reg_off, 0x10C0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_HI_OFFSET + reg_off, 0x10C4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET + reg_off, 0x10C8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) WREG32(mmTPC0_QM_CP_LDMA_COMMIT_OFFSET + reg_off, 0x10CC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) WREG32(mmTPC0_QM_CQ_CFG1 + reg_off, 0x00080008);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) WREG32(mmTPC0_QM_GLBL_ERR_WDATA + reg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) GOYA_ASYNC_EVENT_ID_TPC0_QM + tpc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) WREG32(mmTPC0_QM_GLBL_ERR_CFG + reg_off, QMAN_TPC_ERR_MSG_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) WREG32(mmTPC0_QM_GLBL_PROT + reg_off, QMAN_TPC_ERR_PROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) WREG32(mmTPC0_QM_GLBL_CFG0 + reg_off, QMAN_TPC_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) static void goya_init_tpc_cmdq(struct hl_device *hdev, int tpc_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) u32 mtr_base_lo, mtr_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) u32 gic_base_lo, gic_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) u32 reg_off = tpc_id * (mmTPC1_CMDQ_CQ_CFG1 - mmTPC0_CMDQ_CQ_CFG1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) mtr_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) mtr_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_MON_PAY_ADDRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) gic_base_lo =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) gic_base_hi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_LO + reg_off, mtr_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) WREG32(mmTPC0_CMDQ_CP_MSG_BASE0_ADDR_HI + reg_off, mtr_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_LO + reg_off, so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) WREG32(mmTPC0_CMDQ_CP_MSG_BASE1_ADDR_HI + reg_off, so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) WREG32(mmTPC0_CMDQ_CQ_CFG1 + reg_off, 0x00140014);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_LO + reg_off, gic_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) WREG32(mmTPC0_CMDQ_GLBL_ERR_ADDR_HI + reg_off, gic_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) WREG32(mmTPC0_CMDQ_GLBL_ERR_WDATA + reg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) GOYA_ASYNC_EVENT_ID_TPC0_CMDQ + tpc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) WREG32(mmTPC0_CMDQ_GLBL_ERR_CFG + reg_off, CMDQ_TPC_ERR_MSG_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) WREG32(mmTPC0_CMDQ_GLBL_PROT + reg_off, CMDQ_TPC_ERR_PROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) WREG32(mmTPC0_CMDQ_GLBL_CFG0 + reg_off, CMDQ_TPC_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) void goya_init_tpc_qmans(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) u32 so_base_lo, so_base_hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) u32 cfg_off = mmTPC1_CFG_SM_BASE_ADDRESS_LOW -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) mmTPC0_CFG_SM_BASE_ADDRESS_LOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) if (goya->hw_cap_initialized & HW_CAP_TPC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) so_base_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) so_base_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) for (i = 0 ; i < TPC_MAX_NUM ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_LOW + i * cfg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) so_base_lo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + i * cfg_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) so_base_hi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) goya_init_tpc_qman(hdev, TPC0_QMAN_BASE_OFFSET, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) goya_init_tpc_qman(hdev, TPC1_QMAN_BASE_OFFSET, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) goya_init_tpc_qman(hdev, TPC2_QMAN_BASE_OFFSET, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) goya_init_tpc_qman(hdev, TPC3_QMAN_BASE_OFFSET, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) goya_init_tpc_qman(hdev, TPC4_QMAN_BASE_OFFSET, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) goya_init_tpc_qman(hdev, TPC5_QMAN_BASE_OFFSET, 5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) goya_init_tpc_qman(hdev, TPC6_QMAN_BASE_OFFSET, 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) goya_init_tpc_qman(hdev, TPC7_QMAN_BASE_OFFSET, 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) for (i = 0 ; i < TPC_MAX_NUM ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) goya_init_tpc_cmdq(hdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) goya->hw_cap_initialized |= HW_CAP_TPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) * goya_disable_internal_queues - Disable internal queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) static void goya_disable_internal_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) if (!(goya->hw_cap_initialized & HW_CAP_MME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) goto disable_tpc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) WREG32(mmMME_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) WREG32(mmMME_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) disable_tpc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) if (!(goya->hw_cap_initialized & HW_CAP_TPC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) WREG32(mmTPC0_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) WREG32(mmTPC0_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) WREG32(mmTPC1_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) WREG32(mmTPC1_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) WREG32(mmTPC2_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) WREG32(mmTPC2_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) WREG32(mmTPC3_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) WREG32(mmTPC3_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) WREG32(mmTPC4_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) WREG32(mmTPC4_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) WREG32(mmTPC5_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) WREG32(mmTPC5_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) WREG32(mmTPC6_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) WREG32(mmTPC6_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) WREG32(mmTPC7_QM_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) WREG32(mmTPC7_CMDQ_GLBL_CFG0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) * goya_stop_internal_queues - Stop internal queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) static int goya_stop_internal_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) int rc, retval = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) if (!(goya->hw_cap_initialized & HW_CAP_MME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) goto stop_tpc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) * Each queue (QMAN) is a separate H/W logic. That means that each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) * QMAN can be stopped independently and failure to stop one does NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) * mandate we should not try to stop other QMANs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) mmMME_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) mmMME_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) mmMME_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) dev_err(hdev->dev, "failed to stop MME QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) mmMME_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) mmMME_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) mmMME_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) dev_err(hdev->dev, "failed to stop MME CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) stop_tpc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) if (!(goya->hw_cap_initialized & HW_CAP_TPC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) mmTPC0_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) mmTPC0_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) mmTPC0_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) dev_err(hdev->dev, "failed to stop TPC 0 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) mmTPC0_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) mmTPC0_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) mmTPC0_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) dev_err(hdev->dev, "failed to stop TPC 0 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) mmTPC1_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) mmTPC1_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) mmTPC1_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) dev_err(hdev->dev, "failed to stop TPC 1 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) mmTPC1_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) mmTPC1_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) mmTPC1_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) dev_err(hdev->dev, "failed to stop TPC 1 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) mmTPC2_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) mmTPC2_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) mmTPC2_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) dev_err(hdev->dev, "failed to stop TPC 2 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) mmTPC2_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) mmTPC2_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) mmTPC2_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) dev_err(hdev->dev, "failed to stop TPC 2 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) mmTPC3_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) mmTPC3_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) mmTPC3_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) dev_err(hdev->dev, "failed to stop TPC 3 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) mmTPC3_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) mmTPC3_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) mmTPC3_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) dev_err(hdev->dev, "failed to stop TPC 3 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) mmTPC4_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) mmTPC4_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) mmTPC4_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) dev_err(hdev->dev, "failed to stop TPC 4 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) mmTPC4_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) mmTPC4_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) mmTPC4_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) dev_err(hdev->dev, "failed to stop TPC 4 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) mmTPC5_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) mmTPC5_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) mmTPC5_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) dev_err(hdev->dev, "failed to stop TPC 5 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) mmTPC5_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) mmTPC5_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) mmTPC5_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) dev_err(hdev->dev, "failed to stop TPC 5 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) mmTPC6_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) mmTPC6_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) mmTPC6_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) dev_err(hdev->dev, "failed to stop TPC 6 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) mmTPC6_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) mmTPC6_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) mmTPC6_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) dev_err(hdev->dev, "failed to stop TPC 6 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) mmTPC7_QM_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) mmTPC7_QM_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) mmTPC7_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) dev_err(hdev->dev, "failed to stop TPC 7 QMAN\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) rc = goya_stop_queue(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) mmTPC7_CMDQ_GLBL_CFG1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) mmTPC7_CMDQ_CP_STS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) mmTPC7_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) dev_err(hdev->dev, "failed to stop TPC 7 CMDQ\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) retval = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) static void goya_dma_stall(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) if (!(goya->hw_cap_initialized & HW_CAP_DMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) WREG32(mmDMA_QM_0_GLBL_CFG1, 1 << DMA_QM_0_GLBL_CFG1_DMA_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) WREG32(mmDMA_QM_1_GLBL_CFG1, 1 << DMA_QM_1_GLBL_CFG1_DMA_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) WREG32(mmDMA_QM_2_GLBL_CFG1, 1 << DMA_QM_2_GLBL_CFG1_DMA_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) WREG32(mmDMA_QM_3_GLBL_CFG1, 1 << DMA_QM_3_GLBL_CFG1_DMA_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) WREG32(mmDMA_QM_4_GLBL_CFG1, 1 << DMA_QM_4_GLBL_CFG1_DMA_STOP_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) static void goya_tpc_stall(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) if (!(goya->hw_cap_initialized & HW_CAP_TPC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC1_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC2_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC3_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC4_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC5_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC6_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC7_CFG_TPC_STALL_V_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) static void goya_mme_stall(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) if (!(goya->hw_cap_initialized & HW_CAP_MME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) WREG32(mmMME_STALL, 0xFFFFFFFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) static int goya_enable_msix(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) int cq_cnt = hdev->asic_prop.completion_queues_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) int rc, i, irq_cnt_init, irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) if (goya->hw_cap_initialized & HW_CAP_MSIX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) rc = pci_alloc_irq_vectors(hdev->pdev, GOYA_MSIX_ENTRIES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) GOYA_MSIX_ENTRIES, PCI_IRQ_MSIX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) if (rc < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) "MSI-X: Failed to enable support -- %d/%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) GOYA_MSIX_ENTRIES, rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) for (i = 0, irq_cnt_init = 0 ; i < cq_cnt ; i++, irq_cnt_init++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) irq = pci_irq_vector(hdev->pdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) rc = request_irq(irq, hl_irq_handler_cq, 0, goya_irq_name[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) &hdev->completion_queue[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) dev_err(hdev->dev, "Failed to request IRQ %d", irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) goto free_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) rc = request_irq(irq, hl_irq_handler_eq, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) goya_irq_name[GOYA_EVENT_QUEUE_MSIX_IDX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) &hdev->event_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) dev_err(hdev->dev, "Failed to request IRQ %d", irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) goto free_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) goya->hw_cap_initialized |= HW_CAP_MSIX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) free_irqs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) for (i = 0 ; i < irq_cnt_init ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) free_irq(pci_irq_vector(hdev->pdev, i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) &hdev->completion_queue[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) pci_free_irq_vectors(hdev->pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) static void goya_sync_irqs(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) /* Wait for all pending IRQs to be finished */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) synchronize_irq(pci_irq_vector(hdev->pdev, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) synchronize_irq(pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) static void goya_disable_msix(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) int i, irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) if (!(goya->hw_cap_initialized & HW_CAP_MSIX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) goya_sync_irqs(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) irq = pci_irq_vector(hdev->pdev, GOYA_EVENT_QUEUE_MSIX_IDX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) free_irq(irq, &hdev->event_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) irq = pci_irq_vector(hdev->pdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) free_irq(irq, &hdev->completion_queue[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) pci_free_irq_vectors(hdev->pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) goya->hw_cap_initialized &= ~HW_CAP_MSIX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) static void goya_enable_timestamp(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) /* Disable the timestamp counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) /* Zero the lower/upper parts of the 64-bit counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) /* Enable the counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) static void goya_disable_timestamp(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) /* Disable the timestamp counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) static void goya_halt_engines(struct hl_device *hdev, bool hard_reset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) u32 wait_timeout_ms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) dev_info(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) "Halting compute engines and disabling interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) wait_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) wait_timeout_ms = GOYA_RESET_WAIT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) goya_stop_external_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) goya_stop_internal_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) msleep(wait_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) goya_dma_stall(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) goya_tpc_stall(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) goya_mme_stall(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) msleep(wait_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) goya_disable_external_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) goya_disable_internal_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) goya_disable_timestamp(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) if (hard_reset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) goya_disable_msix(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) goya_mmu_remove_device_cpu_mappings(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) goya_sync_irqs(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) * goya_load_firmware_to_device() - Load LINUX FW code to device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) * @hdev: Pointer to hl_device structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) * Copy LINUX fw code from firmware file to HBM BAR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) * Return: 0 on success, non-zero for failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) static int goya_load_firmware_to_device(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) void __iomem *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) dst = hdev->pcie_bar[DDR_BAR_ID] + LINUX_FW_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) return hl_fw_load_fw_to_device(hdev, GOYA_LINUX_FW_FILE, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) * goya_load_boot_fit_to_device() - Load boot fit to device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) * @hdev: Pointer to hl_device structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) * Copy boot fit file to SRAM BAR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) * Return: 0 on success, non-zero for failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) static int goya_load_boot_fit_to_device(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) void __iomem *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) dst = hdev->pcie_bar[SRAM_CFG_BAR_ID] + BOOT_FIT_SRAM_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) * The version string should be located by that offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) static void goya_read_device_fw_version(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) enum hl_fw_component fwc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) u32 ver_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) char *dest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) switch (fwc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) case FW_COMP_UBOOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) ver_off = RREG32(mmUBOOT_VER_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) dest = hdev->asic_prop.uboot_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) name = "U-Boot";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) case FW_COMP_PREBOOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) ver_off = RREG32(mmPREBOOT_VER_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) dest = hdev->asic_prop.preboot_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) name = "Preboot";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) ver_off &= ~((u32)SRAM_BASE_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) VERSION_MAX_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) name, ver_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) strcpy(dest, "unavailable");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) static int goya_init_cpu(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) if (!hdev->cpu_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) if (goya->hw_cap_initialized & HW_CAP_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) * Before pushing u-boot/linux to device, need to set the ddr bar to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) * base address of dram
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) if (goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) "failed to map DDR bar to DRAM base address\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) mmPSOC_GLOBAL_CONF_UBOOT_MAGIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) mmCPU_CMD_STATUS_TO_HOST, mmCPU_BOOT_ERR0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) false, GOYA_CPU_TIMEOUT_USEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) goya->hw_cap_initialized |= HW_CAP_CPU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) static int goya_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) u64 phys_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) u32 status, timeout_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) WREG32(MMU_ASID_BUSY, 0x80000000 | asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) rc = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) MMU_ASID_BUSY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) !(status & 0x80000000),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) timeout_usec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) "Timeout during MMU hop0 config of asid %d\n", asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) int goya_mmu_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) u64 hop0_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) int rc, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) if (!hdev->mmu_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) if (goya->hw_cap_initialized & HW_CAP_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) hdev->dram_supports_virtual_memory = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) hdev->dram_default_page_mapping = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) for (i = 0 ; i < prop->max_asid ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) hop0_addr = prop->mmu_pgt_addr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) (i * prop->mmu_hop_table_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) rc = goya_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) "failed to set hop0 addr for asid %d\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) goya->hw_cap_initialized |= HW_CAP_MMU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) /* init MMU cache manage page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) WREG32(mmSTLB_CACHE_INV_BASE_39_8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) lower_32_bits(MMU_CACHE_MNG_ADDR >> 8));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) WREG32(mmSTLB_CACHE_INV_BASE_49_40, MMU_CACHE_MNG_ADDR >> 40);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) /* Remove follower feature due to performance bug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) WREG32_AND(mmSTLB_STLB_FEATURE_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) (~STLB_STLB_FEATURE_EN_FOLLOWER_EN_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) VM_TYPE_USERPTR | VM_TYPE_PHYS_PACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) WREG32(mmMMU_MMU_ENABLE, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) WREG32(mmMMU_SPI_MASK, 0xF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) * goya_hw_init - Goya hardware initialization code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) static int goya_hw_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) dev_info(hdev->dev, "Starting initialization of H/W\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) /* Perform read from the device to make sure device is up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) * Let's mark in the H/W that we have reached this point. We check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) * this value in the reset_before_init function to understand whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) * we need to reset the chip before doing H/W init. This register is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) * cleared by the H/W upon H/W reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) rc = goya_init_cpu(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) dev_err(hdev->dev, "failed to initialize CPU\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) goya_tpc_mbist_workaround(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) goya_init_golden_registers(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) * After CPU initialization is finished, change DDR bar mapping inside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) * iATU to point to the start address of the MMU page tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) if (goya_set_ddr_bar_base(hdev, (MMU_PAGE_TABLES_ADDR &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) ~(prop->dram_pci_bar_size - 0x1ull))) == U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) "failed to map DDR bar to MMU page tables\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) rc = goya_mmu_init(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) goya_init_security(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) goya_init_dma_qmans(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) goya_init_mme_qmans(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) goya_init_tpc_qmans(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) goya_enable_timestamp(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) /* MSI-X must be enabled before CPU queues are initialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) rc = goya_enable_msix(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) goto disable_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) /* Perform read from the device to flush all MSI-X configuration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) RREG32(mmPCIE_DBI_DEVICE_ID_VENDOR_ID_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) disable_queues:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) goya_disable_internal_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) goya_disable_external_queues(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) * goya_hw_fini - Goya hardware tear-down code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) * @hard_reset: should we do hard reset to all engines or just reset the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) * compute/dma engines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) static void goya_hw_fini(struct hl_device *hdev, bool hard_reset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) u32 reset_timeout_ms, cpu_timeout_ms, status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) if (hdev->pldm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) reset_timeout_ms = GOYA_PLDM_RESET_TIMEOUT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) cpu_timeout_ms = GOYA_PLDM_RESET_WAIT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) reset_timeout_ms = GOYA_RESET_TIMEOUT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) cpu_timeout_ms = GOYA_CPU_RESET_WAIT_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) if (hard_reset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) /* I don't know what is the state of the CPU so make sure it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) * stopped in any means necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) WREG32(mmPSOC_GLOBAL_CONF_UBOOT_MAGIC, KMD_MSG_GOTO_WFE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) GOYA_ASYNC_EVENT_ID_HALT_MACHINE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) msleep(cpu_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) goya_set_ddr_bar_base(hdev, DRAM_PHYS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) goya_disable_clk_rlx(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) goya_set_pll_refclk(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, RESET_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) dev_info(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) "Issued HARD reset command, going to wait %dms\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) reset_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG, DMA_MME_TPC_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) dev_info(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) "Issued SOFT reset command, going to wait %dms\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) reset_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) * After hard reset, we can't poll the BTM_FSM register because the PSOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) * itself is in reset. In either reset we need to wait until the reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) * is deasserted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) msleep(reset_timeout_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) "Timeout while waiting for device to reset 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) if (!hard_reset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) goya->hw_cap_initialized &= ~(HW_CAP_DMA | HW_CAP_MME |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) HW_CAP_GOLDEN | HW_CAP_TPC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) GOYA_ASYNC_EVENT_ID_SOFT_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) /* Chicken bit to re-initiate boot sequencer flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) 1 << PSOC_GLOBAL_CONF_BOOT_SEQ_RE_START_IND_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) /* Move boot manager FSM to pre boot sequencer init state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) WREG32(mmPSOC_GLOBAL_CONF_SW_BTM_FSM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) 0xA << PSOC_GLOBAL_CONF_SW_BTM_FSM_CTRL_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) goya->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) HW_CAP_DDR_0 | HW_CAP_DDR_1 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) HW_CAP_DMA | HW_CAP_MME |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) HW_CAP_MMU | HW_CAP_TPC_MBIST |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) HW_CAP_GOLDEN | HW_CAP_TPC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) memset(goya->events_stat, 0, sizeof(goya->events_stat));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) int goya_suspend(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) dev_err(hdev->dev, "Failed to disable PCI access from CPU\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) int goya_resume(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) return goya_init_iatu(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) static int goya_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) void *cpu_addr, dma_addr_t dma_addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) VM_DONTCOPY | VM_NORESERVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) (dma_addr - HOST_PHYS_BASE), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) void goya_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) u32 db_reg_offset, db_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) switch (hw_queue_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) case GOYA_QUEUE_ID_DMA_0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) db_reg_offset = mmDMA_QM_0_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) case GOYA_QUEUE_ID_DMA_1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) db_reg_offset = mmDMA_QM_1_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) case GOYA_QUEUE_ID_DMA_2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) db_reg_offset = mmDMA_QM_2_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) case GOYA_QUEUE_ID_DMA_3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) db_reg_offset = mmDMA_QM_3_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) case GOYA_QUEUE_ID_DMA_4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) db_reg_offset = mmDMA_QM_4_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) case GOYA_QUEUE_ID_CPU_PQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) db_reg_offset = mmCPU_IF_PF_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) case GOYA_QUEUE_ID_MME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) db_reg_offset = mmMME_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) case GOYA_QUEUE_ID_TPC0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) db_reg_offset = mmTPC0_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) case GOYA_QUEUE_ID_TPC1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) db_reg_offset = mmTPC1_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) case GOYA_QUEUE_ID_TPC2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) db_reg_offset = mmTPC2_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) case GOYA_QUEUE_ID_TPC3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) db_reg_offset = mmTPC3_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) case GOYA_QUEUE_ID_TPC4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) db_reg_offset = mmTPC4_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) case GOYA_QUEUE_ID_TPC5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) db_reg_offset = mmTPC5_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) case GOYA_QUEUE_ID_TPC6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) db_reg_offset = mmTPC6_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) case GOYA_QUEUE_ID_TPC7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) db_reg_offset = mmTPC7_QM_PQ_PI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) /* Should never get here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) dev_err(hdev->dev, "H/W queue %d is invalid. Can't set pi\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) hw_queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) db_value = pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) /* ring the doorbell */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) WREG32(db_reg_offset, db_value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) if (hw_queue_id == GOYA_QUEUE_ID_CPU_PQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) GOYA_ASYNC_EVENT_ID_PI_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) void goya_pqe_write(struct hl_device *hdev, __le64 *pqe, struct hl_bd *bd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) /* The QMANs are on the SRAM so need to copy to IO space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) memcpy_toio((void __iomem *) pqe, bd, sizeof(struct hl_bd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) static void *goya_dma_alloc_coherent(struct hl_device *hdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) dma_addr_t *dma_handle, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) dma_handle, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) /* Shift to the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) if (kernel_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) *dma_handle += HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) return kernel_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) static void goya_dma_free_coherent(struct hl_device *hdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) void *cpu_addr, dma_addr_t dma_handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) /* Cancel the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) dma_addr_t *dma_handle, u16 *queue_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) void *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) u32 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) *dma_handle = hdev->asic_prop.sram_base_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) switch (queue_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) case GOYA_QUEUE_ID_MME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) offset = MME_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) *queue_len = MME_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) case GOYA_QUEUE_ID_TPC0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) offset = TPC0_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) case GOYA_QUEUE_ID_TPC1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) offset = TPC1_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) case GOYA_QUEUE_ID_TPC2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) offset = TPC2_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) case GOYA_QUEUE_ID_TPC3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) offset = TPC3_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) case GOYA_QUEUE_ID_TPC4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) offset = TPC4_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) case GOYA_QUEUE_ID_TPC5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) offset = TPC5_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) case GOYA_QUEUE_ID_TPC6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) offset = TPC6_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) case GOYA_QUEUE_ID_TPC7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) offset = TPC7_QMAN_BASE_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) *queue_len = TPC_QMAN_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) base += offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) *dma_handle += offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) return base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) static int goya_send_job_on_qman0(struct hl_device *hdev, struct hl_cs_job *job)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) struct packet_msg_prot *fence_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) u32 *fence_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) dma_addr_t fence_dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) struct hl_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) u32 tmp, timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) timeout = GOYA_PLDM_QMAN0_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) timeout = HL_DEVICE_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) if (!hdev->asic_funcs->is_device_idle(hdev, NULL, NULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) dev_err_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) "Can't send driver job on QMAN0 because the device is not idle\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) &fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) if (!fence_ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) "Failed to allocate fence memory for QMAN0\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) goya_qman0_set_security(hdev, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) cb = job->patched_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) fence_pkt = cb->kernel_address +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) job->job_cb_size - sizeof(struct packet_msg_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) (1 << GOYA_PKT_CTL_EB_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) (1 << GOYA_PKT_CTL_MB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) fence_pkt->ctl = cpu_to_le32(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) fence_pkt->value = cpu_to_le32(GOYA_QMAN0_FENCE_VAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) fence_pkt->addr = cpu_to_le64(fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) rc = hl_hw_queue_send_cb_no_cmpl(hdev, GOYA_QUEUE_ID_DMA_0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) job->job_cb_size, cb->bus_address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) goto free_fence_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) (tmp == GOYA_QMAN0_FENCE_VAL), 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) timeout, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) hl_hw_queue_inc_ci_kernel(hdev, GOYA_QUEUE_ID_DMA_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) if (rc == -ETIMEDOUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) goto free_fence_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) free_fence_ptr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) goya_qman0_set_security(hdev, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) int goya_send_cpu_message(struct hl_device *hdev, u32 *msg, u16 len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) u32 timeout, long *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) *result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) if (!timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) timeout = GOYA_MSG_TO_CPU_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) return hl_fw_send_cpu_message(hdev, GOYA_QUEUE_ID_CPU_PQ, msg, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) timeout, result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) int goya_test_queue(struct hl_device *hdev, u32 hw_queue_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) struct packet_msg_prot *fence_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) dma_addr_t pkt_dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) u32 fence_val, tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) dma_addr_t fence_dma_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) u32 *fence_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) fence_val = GOYA_QMAN0_FENCE_VAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) fence_ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) &fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) if (!fence_ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) "Failed to allocate memory for H/W queue %d testing\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) hw_queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) *fence_ptr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) fence_pkt = hdev->asic_funcs->asic_dma_pool_zalloc(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) sizeof(struct packet_msg_prot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) GFP_KERNEL, &pkt_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) if (!fence_pkt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) "Failed to allocate packet for H/W queue %d testing\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) hw_queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) goto free_fence_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) (1 << GOYA_PKT_CTL_EB_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) (1 << GOYA_PKT_CTL_MB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) fence_pkt->ctl = cpu_to_le32(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) fence_pkt->value = cpu_to_le32(fence_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) fence_pkt->addr = cpu_to_le64(fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) sizeof(struct packet_msg_prot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) pkt_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) "Failed to send fence packet to H/W queue %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) hw_queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) goto free_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) 1000, GOYA_TEST_QUEUE_WAIT_USEC, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) if (rc == -ETIMEDOUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) "H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) hw_queue_id, (unsigned long long) fence_dma_addr, tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) rc = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) free_pkt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) pkt_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) free_fence_ptr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) hdev->asic_funcs->asic_dma_pool_free(hdev, (void *) fence_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) fence_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) int goya_test_cpu_queue(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) * check capability here as send_cpu_message() won't update the result
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) * value if no capability
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) return hl_fw_test_cpu_queue(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) int goya_test_queues(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) int i, rc, ret_val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) rc = goya_test_queue(hdev, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) ret_val = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) return ret_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) static void *goya_dma_pool_zalloc(struct hl_device *hdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) gfp_t mem_flags, dma_addr_t *dma_handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) void *kernel_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) if (size > GOYA_DMA_POOL_BLK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) /* Shift to the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) if (kernel_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) *dma_handle += HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) return kernel_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) static void goya_dma_pool_free(struct hl_device *hdev, void *vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) dma_addr_t dma_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) /* Cancel the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) void *goya_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) dma_addr_t *dma_handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) vaddr = hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) *dma_handle = (*dma_handle) - hdev->cpu_accessible_dma_address +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) VA_CPU_ACCESSIBLE_MEM_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) return vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) void goya_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) void *vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) static int goya_dma_map_sg(struct hl_device *hdev, struct scatterlist *sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) int nents, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) if (!dma_map_sg(&hdev->pdev->dev, sgl, nents, dir))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) /* Shift to the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) for_each_sg(sgl, sg, nents, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) sg->dma_address += HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) static void goya_dma_unmap_sg(struct hl_device *hdev, struct scatterlist *sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) int nents, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) /* Cancel the device's base physical address of host memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) for_each_sg(sgl, sg, nents, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) sg->dma_address -= HOST_PHYS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) dma_unmap_sg(&hdev->pdev->dev, sgl, nents, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) u32 goya_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) struct scatterlist *sg, *sg_next_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) u32 count, dma_desc_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) u64 len, len_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) dma_addr_t addr, addr_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) dma_desc_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) for_each_sg(sgt->sgl, sg, sgt->nents, count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) len = sg_dma_len(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) addr = sg_dma_address(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) if (len == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) while ((count + 1) < sgt->nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) sg_next_iter = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) len_next = sg_dma_len(sg_next_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) addr_next = sg_dma_address(sg_next_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) if (len_next == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) if ((addr + len == addr_next) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) len += len_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) sg = sg_next_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) dma_desc_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) return dma_desc_cnt * sizeof(struct packet_lin_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) static int goya_pin_memory_before_cs(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) struct packet_lin_dma *user_dma_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) u64 addr, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) struct hl_userptr *userptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) parser->job_userptr_list, &userptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) goto already_pinned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) userptr = kzalloc(sizeof(*userptr), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) if (!userptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) userptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) goto free_userptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) list_add_tail(&userptr->job_node, parser->job_userptr_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) rc = hdev->asic_funcs->asic_dma_map_sg(hdev, userptr->sgt->sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) userptr->sgt->nents, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) dev_err(hdev->dev, "failed to map sgt with DMA region\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) goto unpin_memory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) userptr->dma_mapped = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) userptr->dir = dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) already_pinned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) parser->patched_cb_size +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) goya_get_dma_desc_list_size(hdev, userptr->sgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) unpin_memory:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) list_del(&userptr->job_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) hl_unpin_host_memory(hdev, userptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) free_userptr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) kfree(userptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) static int goya_validate_dma_pkt_host(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) struct packet_lin_dma *user_dma_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) u64 device_memory_addr, addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) enum dma_data_direction dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) enum goya_dma_direction user_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) bool sram_addr = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) bool skip_host_mem_pin = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) bool user_memset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) u32 ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) ctl = le32_to_cpu(user_dma_pkt->ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) switch (user_dir) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) case DMA_HOST_TO_DRAM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) dev_dbg(hdev->dev, "DMA direction is HOST --> DRAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) sram_addr = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) if (user_memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) skip_host_mem_pin = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) case DMA_DRAM_TO_HOST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) dev_dbg(hdev->dev, "DMA direction is DRAM --> HOST\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) dir = DMA_FROM_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) sram_addr = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) case DMA_HOST_TO_SRAM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) dev_dbg(hdev->dev, "DMA direction is HOST --> SRAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) if (user_memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) skip_host_mem_pin = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) case DMA_SRAM_TO_HOST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) dev_dbg(hdev->dev, "DMA direction is SRAM --> HOST\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) dir = DMA_FROM_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) dev_err(hdev->dev, "DMA direction is undefined\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) if (sram_addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) if (!hl_mem_area_inside_range(device_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) hdev->asic_prop.sram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) hdev->asic_prop.sram_end_address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) "SRAM address 0x%llx + 0x%x is invalid\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) device_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) user_dma_pkt->tsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) if (!hl_mem_area_inside_range(device_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) hdev->asic_prop.dram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) hdev->asic_prop.dram_end_address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) "DRAM address 0x%llx + 0x%x is invalid\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) device_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) user_dma_pkt->tsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) if (skip_host_mem_pin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) parser->patched_cb_size += sizeof(*user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) if ((dir == DMA_TO_DEVICE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) (parser->hw_queue_id > GOYA_QUEUE_ID_DMA_1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) "Can't DMA from host on queue other then 1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) rc = goya_pin_memory_before_cs(hdev, parser, user_dma_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) addr, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) static int goya_validate_dma_pkt_no_host(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) struct packet_lin_dma *user_dma_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) u64 sram_memory_addr, dram_memory_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) enum goya_dma_direction user_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) u32 ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) ctl = le32_to_cpu(user_dma_pkt->ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) if (user_dir == DMA_DRAM_TO_SRAM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) dev_dbg(hdev->dev, "DMA direction is DRAM --> SRAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) dram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) sram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) dev_dbg(hdev->dev, "DMA direction is SRAM --> DRAM\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) sram_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) dram_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) if (!hl_mem_area_inside_range(sram_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) hdev->asic_prop.sram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) hdev->asic_prop.sram_end_address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) dev_err(hdev->dev, "SRAM address 0x%llx + 0x%x is invalid\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) sram_memory_addr, user_dma_pkt->tsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) if (!hl_mem_area_inside_range(dram_memory_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) hdev->asic_prop.dram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) hdev->asic_prop.dram_end_address)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) dev_err(hdev->dev, "DRAM address 0x%llx + 0x%x is invalid\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) dram_memory_addr, user_dma_pkt->tsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) parser->patched_cb_size += sizeof(*user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) static int goya_validate_dma_pkt_no_mmu(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) struct packet_lin_dma *user_dma_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) enum goya_dma_direction user_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) u32 ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) dev_dbg(hdev->dev, "DMA packet details:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) dev_dbg(hdev->dev, "source == 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) le64_to_cpu(user_dma_pkt->src_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) dev_dbg(hdev->dev, "destination == 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) le64_to_cpu(user_dma_pkt->dst_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) ctl = le32_to_cpu(user_dma_pkt->ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) * Special handling for DMA with size 0. The H/W has a bug where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) * this can cause the QMAN DMA to get stuck, so block it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) if (user_dma_pkt->tsize == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) "Got DMA with size 0, might reset the device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) rc = goya_validate_dma_pkt_no_host(hdev, parser, user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) rc = goya_validate_dma_pkt_host(hdev, parser, user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) static int goya_validate_dma_pkt_mmu(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) struct packet_lin_dma *user_dma_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) dev_dbg(hdev->dev, "DMA packet details:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) dev_dbg(hdev->dev, "source == 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) le64_to_cpu(user_dma_pkt->src_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) dev_dbg(hdev->dev, "destination == 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) le64_to_cpu(user_dma_pkt->dst_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) * WA for HW-23.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) * We can't allow user to read from Host using QMANs other than 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) * PMMU and HPMMU addresses are equal, check only one of them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) if (parser->hw_queue_id != GOYA_QUEUE_ID_DMA_1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) hl_mem_area_inside_range(le64_to_cpu(user_dma_pkt->src_addr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) hdev->asic_prop.pmmu.start_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) hdev->asic_prop.pmmu.end_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) "Can't DMA from host on queue other then 1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) if (user_dma_pkt->tsize == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) "Got DMA with size 0, might reset the device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) parser->patched_cb_size += sizeof(*user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) static int goya_validate_wreg32(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) struct packet_wreg32 *wreg_pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) u32 sob_start_addr, sob_end_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) u16 reg_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) reg_offset = le32_to_cpu(wreg_pkt->ctl) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) GOYA_PKT_WREG32_CTL_REG_OFFSET_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) dev_dbg(hdev->dev, "WREG32 packet details:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) dev_dbg(hdev->dev, "reg_offset == 0x%x\n", reg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) dev_dbg(hdev->dev, "value == 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) le32_to_cpu(wreg_pkt->value));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) if (reg_offset != (mmDMA_CH_0_WR_COMP_ADDR_LO & 0x1FFF)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) dev_err(hdev->dev, "WREG32 packet with illegal address 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) reg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) * With MMU, DMA channels are not secured, so it doesn't matter where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) * the WR COMP will be written to because it will go out with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) * non-secured property
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) if (goya->hw_cap_initialized & HW_CAP_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) sob_start_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) sob_end_addr = lower_32_bits(CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1023);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) if ((le32_to_cpu(wreg_pkt->value) < sob_start_addr) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) (le32_to_cpu(wreg_pkt->value) > sob_end_addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) dev_err(hdev->dev, "WREG32 packet with illegal value 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) wreg_pkt->value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) static int goya_validate_cb(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) struct hl_cs_parser *parser, bool is_mmu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) u32 cb_parsed_length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) parser->patched_cb_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) /* cb_user_size is more than 0 so loop will always be executed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) while (cb_parsed_length < parser->user_cb_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) enum packet_id pkt_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) u16 pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) struct goya_packet *user_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) user_pkt = parser->user_cb->kernel_address + cb_parsed_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) pkt_id = (enum packet_id) (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) (le64_to_cpu(user_pkt->header) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) PACKET_HEADER_PACKET_ID_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) PACKET_HEADER_PACKET_ID_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) if (!validate_packet_id(pkt_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) pkt_size = goya_packet_sizes[pkt_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) cb_parsed_length += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) if (cb_parsed_length > parser->user_cb_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) "packet 0x%x is out of CB boundary\n", pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) switch (pkt_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) case PACKET_WREG_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) * Although it is validated after copy in patch_cb(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) * need to validate here as well because patch_cb() is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) * not called in MMU path while this function is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) rc = goya_validate_wreg32(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) parser, (struct packet_wreg32 *) user_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) parser->patched_cb_size += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) case PACKET_WREG_BULK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) "User not allowed to use WREG_BULK\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) case PACKET_MSG_PROT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) "User not allowed to use MSG_PROT\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) case PACKET_CP_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) case PACKET_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) dev_err(hdev->dev, "User not allowed to use STOP\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) case PACKET_LIN_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) if (is_mmu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) rc = goya_validate_dma_pkt_mmu(hdev, parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) (struct packet_lin_dma *) user_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) rc = goya_validate_dma_pkt_no_mmu(hdev, parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) (struct packet_lin_dma *) user_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) case PACKET_MSG_LONG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) case PACKET_MSG_SHORT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) case PACKET_FENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) case PACKET_NOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) parser->patched_cb_size += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) dev_err(hdev->dev, "Invalid packet header 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) * The new CB should have space at the end for two MSG_PROT packets:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) * 1. A packet that will act as a completion packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) * 2. A packet that will generate MSI-X interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) parser->patched_cb_size += sizeof(struct packet_msg_prot) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) static int goya_patch_dma_packet(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) struct hl_cs_parser *parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) struct packet_lin_dma *user_dma_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) struct packet_lin_dma *new_dma_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) u32 *new_dma_pkt_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) struct hl_userptr *userptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) struct scatterlist *sg, *sg_next_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) u32 count, dma_desc_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) u64 len, len_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) dma_addr_t dma_addr, dma_addr_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) enum goya_dma_direction user_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) u64 device_memory_addr, addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) enum dma_data_direction dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) struct sg_table *sgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) bool skip_host_mem_pin = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) bool user_memset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) u32 user_rdcomp_mask, user_wrcomp_mask, ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) ctl = le32_to_cpu(user_dma_pkt->ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) user_dir = (ctl & GOYA_PKT_LIN_DMA_CTL_DMA_DIR_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) user_memset = (ctl & GOYA_PKT_LIN_DMA_CTL_MEMSET_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) if ((user_dir == DMA_DRAM_TO_SRAM) || (user_dir == DMA_SRAM_TO_DRAM) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) (user_dma_pkt->tsize == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) memcpy(new_dma_pkt, user_dma_pkt, sizeof(*new_dma_pkt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) *new_dma_pkt_size = sizeof(*new_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) if ((user_dir == DMA_HOST_TO_DRAM) || (user_dir == DMA_HOST_TO_SRAM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) dir = DMA_TO_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) if (user_memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) skip_host_mem_pin = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) addr = le64_to_cpu(user_dma_pkt->dst_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) dir = DMA_FROM_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) if ((!skip_host_mem_pin) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) (hl_userptr_is_pinned(hdev, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) le32_to_cpu(user_dma_pkt->tsize),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) parser->job_userptr_list, &userptr) == false)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) addr, user_dma_pkt->tsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) if ((user_memset) && (dir == DMA_TO_DEVICE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) *new_dma_pkt_size = sizeof(*user_dma_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) user_rdcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) user_wrcomp_mask = ctl & GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) sgt = userptr->sgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) dma_desc_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) for_each_sg(sgt->sgl, sg, sgt->nents, count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) len = sg_dma_len(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) dma_addr = sg_dma_address(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) if (len == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) while ((count + 1) < sgt->nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) sg_next_iter = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) len_next = sg_dma_len(sg_next_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) dma_addr_next = sg_dma_address(sg_next_iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) if (len_next == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) if ((dma_addr + len == dma_addr_next) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) (len + len_next <= DMA_MAX_TRANSFER_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) len += len_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) sg = sg_next_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) ctl = le32_to_cpu(user_dma_pkt->ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) if (likely(dma_desc_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) ctl &= ~GOYA_PKT_CTL_EB_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) ctl &= ~(GOYA_PKT_LIN_DMA_CTL_RDCOMP_MASK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) GOYA_PKT_LIN_DMA_CTL_WRCOMP_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) new_dma_pkt->ctl = cpu_to_le32(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) new_dma_pkt->tsize = cpu_to_le32((u32) len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) if (dir == DMA_TO_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) new_dma_pkt->src_addr = cpu_to_le64(dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) if (!user_memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) device_memory_addr += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) dma_desc_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) new_dma_pkt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) if (!dma_desc_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) "Error of 0 SG entries when patching DMA packet\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) /* Fix the last dma packet - rdcomp/wrcomp must be as user set them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) new_dma_pkt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) new_dma_pkt->ctl |= cpu_to_le32(user_rdcomp_mask | user_wrcomp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) *new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) static int goya_patch_cb(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) struct hl_cs_parser *parser)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) u32 cb_parsed_length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) u32 cb_patched_cur_length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) /* cb_user_size is more than 0 so loop will always be executed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) while (cb_parsed_length < parser->user_cb_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) enum packet_id pkt_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) u16 pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) u32 new_pkt_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) struct goya_packet *user_pkt, *kernel_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) user_pkt = parser->user_cb->kernel_address + cb_parsed_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) kernel_pkt = parser->patched_cb->kernel_address +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) cb_patched_cur_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) pkt_id = (enum packet_id) (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) (le64_to_cpu(user_pkt->header) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) PACKET_HEADER_PACKET_ID_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) PACKET_HEADER_PACKET_ID_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) if (!validate_packet_id(pkt_id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) pkt_size = goya_packet_sizes[pkt_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) cb_parsed_length += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) if (cb_parsed_length > parser->user_cb_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) "packet 0x%x is out of CB boundary\n", pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) switch (pkt_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) case PACKET_LIN_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) rc = goya_patch_dma_packet(hdev, parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) (struct packet_lin_dma *) user_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) (struct packet_lin_dma *) kernel_pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) &new_pkt_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) cb_patched_cur_length += new_pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) case PACKET_WREG_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) memcpy(kernel_pkt, user_pkt, pkt_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) cb_patched_cur_length += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) rc = goya_validate_wreg32(hdev, parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) (struct packet_wreg32 *) kernel_pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) case PACKET_WREG_BULK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) "User not allowed to use WREG_BULK\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) case PACKET_MSG_PROT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) "User not allowed to use MSG_PROT\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) case PACKET_CP_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) dev_err(hdev->dev, "User not allowed to use CP_DMA\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) case PACKET_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) dev_err(hdev->dev, "User not allowed to use STOP\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) rc = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) case PACKET_MSG_LONG:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) case PACKET_MSG_SHORT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) case PACKET_FENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) case PACKET_NOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) memcpy(kernel_pkt, user_pkt, pkt_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) cb_patched_cur_length += pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) dev_err(hdev->dev, "Invalid packet header 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) pkt_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) static int goya_parse_cb_mmu(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) struct hl_cs_parser *parser)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) u64 patched_cb_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) u32 patched_cb_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) struct hl_cb *user_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) * The new CB should have space at the end for two MSG_PROT pkt:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) * 1. A packet that will act as a completion packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) * 2. A packet that will generate MSI-X interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) parser->patched_cb_size = parser->user_cb_size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) sizeof(struct packet_msg_prot) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) parser->patched_cb_size, false, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) &patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) "Failed to allocate patched CB for DMA CS %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) patched_cb_handle >>= PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) (u32) patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) /* hl_cb_get should never fail here so use kernel WARN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) (u32) patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) if (!parser->patched_cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) * The check that parser->user_cb_size <= parser->user_cb->size was done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) * in validate_queue_index().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) memcpy(parser->patched_cb->kernel_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) parser->user_cb->kernel_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) parser->user_cb_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) patched_cb_size = parser->patched_cb_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) /* validate patched CB instead of user CB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) user_cb = parser->user_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) parser->user_cb = parser->patched_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) rc = goya_validate_cb(hdev, parser, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) parser->user_cb = user_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) hl_cb_put(parser->patched_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) if (patched_cb_size != parser->patched_cb_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) dev_err(hdev->dev, "user CB size mismatch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) hl_cb_put(parser->patched_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) * Always call cb destroy here because we still have 1 reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) * to it by calling cb_get earlier. After the job will be completed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) * cb_put will release it, but here we want to remove it from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) * idr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) patched_cb_handle << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) static int goya_parse_cb_no_mmu(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) struct hl_cs_parser *parser)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) u64 patched_cb_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) rc = goya_validate_cb(hdev, parser, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) goto free_userptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) parser->patched_cb_size, false, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) &patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) "Failed to allocate patched CB for DMA CS %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) goto free_userptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) patched_cb_handle >>= PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) parser->patched_cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) (u32) patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) /* hl_cb_get should never fail here so use kernel WARN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) WARN(!parser->patched_cb, "DMA CB handle invalid 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) (u32) patched_cb_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) if (!parser->patched_cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) rc = goya_patch_cb(hdev, parser);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) hl_cb_put(parser->patched_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) * Always call cb destroy here because we still have 1 reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) * to it by calling cb_get earlier. After the job will be completed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) * cb_put will release it, but here we want to remove it from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) * idr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) hl_cb_destroy(hdev, &hdev->kernel_cb_mgr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) patched_cb_handle << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) free_userptr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) hl_userptr_delete_list(hdev, parser->job_userptr_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) static int goya_parse_cb_no_ext_queue(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) struct hl_cs_parser *parser)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) struct asic_fixed_properties *asic_prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) if (goya->hw_cap_initialized & HW_CAP_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) /* For internal queue jobs, just check if CB address is valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) if (hl_mem_area_inside_range(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) (u64) (uintptr_t) parser->user_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) parser->user_cb_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) asic_prop->sram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) asic_prop->sram_end_address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) if (hl_mem_area_inside_range(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) (u64) (uintptr_t) parser->user_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) parser->user_cb_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) asic_prop->dram_user_base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) asic_prop->dram_end_address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) "Internal CB address 0x%px + 0x%x is not in SRAM nor in DRAM\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) parser->user_cb, parser->user_cb_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) int goya_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) if (parser->queue_type == QUEUE_TYPE_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) return goya_parse_cb_no_ext_queue(hdev, parser);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) if (goya->hw_cap_initialized & HW_CAP_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) return goya_parse_cb_mmu(hdev, parser);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) return goya_parse_cb_no_mmu(hdev, parser);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) void goya_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) u32 len, u64 cq_addr, u32 cq_val, u32 msix_vec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) bool eb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) struct packet_msg_prot *cq_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) u32 tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) (1 << GOYA_PKT_CTL_EB_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) (1 << GOYA_PKT_CTL_MB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) cq_pkt->ctl = cpu_to_le32(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) cq_pkt->value = cpu_to_le32(cq_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) cq_pkt->addr = cpu_to_le64(cq_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) cq_pkt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) tmp = (PACKET_MSG_PROT << GOYA_PKT_CTL_OPCODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) (1 << GOYA_PKT_CTL_MB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) cq_pkt->ctl = cpu_to_le32(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) cq_pkt->value = cpu_to_le32(msix_vec & 0x7FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) cq_pkt->addr = cpu_to_le64(CFG_BASE + mmPCIE_DBI_MSIX_DOORBELL_OFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) void goya_update_eq_ci(struct hl_device *hdev, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) WREG32(mmCPU_EQ_CI, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) void goya_restore_phase_topology(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) static void goya_clear_sm_regs(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) int i, num_of_sob_in_longs, num_of_mon_in_longs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) num_of_sob_in_longs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) ((mmSYNC_MNGR_SOB_OBJ_1023 - mmSYNC_MNGR_SOB_OBJ_0) + 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) num_of_mon_in_longs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) ((mmSYNC_MNGR_MON_STATUS_255 - mmSYNC_MNGR_MON_STATUS_0) + 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) for (i = 0 ; i < num_of_sob_in_longs ; i += 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) WREG32(mmSYNC_MNGR_SOB_OBJ_0 + i, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) for (i = 0 ; i < num_of_mon_in_longs ; i += 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) WREG32(mmSYNC_MNGR_MON_STATUS_0 + i, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) /* Flush all WREG to prevent race */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) i = RREG32(mmSYNC_MNGR_SOB_OBJ_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) * goya_debugfs_read32 - read a 32bit value from a given device or a host mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) * address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) * @addr: device or host mapped address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) * @val: returned value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) * In case of DDR address that is not mapped into the default aperture that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) * the DDR bar exposes, the function will configure the iATU so that the DDR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) * bar will be positioned at a base address that allows reading from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) * required address. Configuring the iATU during normal operation can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) * lead to undefined behavior and therefore, should be done with extreme care
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) static int goya_debugfs_read32(struct hl_device *hdev, u64 addr, u32 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) u64 ddr_bar_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) *val = RREG32(addr - CFG_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) } else if ((addr >= SRAM_BASE_ADDR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) *val = readl(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) (addr - SRAM_BASE_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) u64 bar_base_addr = DRAM_PHYS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) (addr & ~(prop->dram_pci_bar_size - 0x1ull));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) if (ddr_bar_addr != U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) *val = readl(hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) (addr - bar_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) ddr_bar_addr = goya_set_ddr_bar_base(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) ddr_bar_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) if (ddr_bar_addr == U64_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) rc = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) *val = *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) * goya_debugfs_write32 - write a 32bit value to a given device or a host mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) * address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) * @hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) * @addr: device or host mapped address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) * @val: returned value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) * In case of DDR address that is not mapped into the default aperture that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) * the DDR bar exposes, the function will configure the iATU so that the DDR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) * bar will be positioned at a base address that allows writing to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) * required address. Configuring the iATU during normal operation can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) * lead to undefined behavior and therefore, should be done with extreme care
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) static int goya_debugfs_write32(struct hl_device *hdev, u64 addr, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) u64 ddr_bar_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) if ((addr >= CFG_BASE) && (addr < CFG_BASE + CFG_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) WREG32(addr - CFG_BASE, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) } else if ((addr >= SRAM_BASE_ADDR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) (addr < SRAM_BASE_ADDR + SRAM_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) writel(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) (addr - SRAM_BASE_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) } else if (addr < DRAM_PHYS_BASE + hdev->asic_prop.dram_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) u64 bar_base_addr = DRAM_PHYS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) (addr & ~(prop->dram_pci_bar_size - 0x1ull));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) if (ddr_bar_addr != U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) writel(val, hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) (addr - bar_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) ddr_bar_addr = goya_set_ddr_bar_base(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) ddr_bar_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) if (ddr_bar_addr == U64_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) rc = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) *(u32 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) static int goya_debugfs_read64(struct hl_device *hdev, u64 addr, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) u64 ddr_bar_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) u32 val_l = RREG32(addr - CFG_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) u32 val_h = RREG32(addr + sizeof(u32) - CFG_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) *val = (((u64) val_h) << 32) | val_l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) } else if ((addr >= SRAM_BASE_ADDR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) *val = readq(hdev->pcie_bar[SRAM_CFG_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) (addr - SRAM_BASE_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) } else if (addr <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) u64 bar_base_addr = DRAM_PHYS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) (addr & ~(prop->dram_pci_bar_size - 0x1ull));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) if (ddr_bar_addr != U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) *val = readq(hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) (addr - bar_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) ddr_bar_addr = goya_set_ddr_bar_base(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) ddr_bar_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) if (ddr_bar_addr == U64_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) rc = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) *val = *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) static int goya_debugfs_write64(struct hl_device *hdev, u64 addr, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) u64 ddr_bar_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) if ((addr >= CFG_BASE) && (addr <= CFG_BASE + CFG_SIZE - sizeof(u64))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) WREG32(addr - CFG_BASE, lower_32_bits(val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) WREG32(addr + sizeof(u32) - CFG_BASE, upper_32_bits(val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) } else if ((addr >= SRAM_BASE_ADDR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) (addr <= SRAM_BASE_ADDR + SRAM_SIZE - sizeof(u64))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) writeq(val, hdev->pcie_bar[SRAM_CFG_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) (addr - SRAM_BASE_ADDR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) } else if (addr <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) DRAM_PHYS_BASE + hdev->asic_prop.dram_size - sizeof(u64)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) u64 bar_base_addr = DRAM_PHYS_BASE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) (addr & ~(prop->dram_pci_bar_size - 0x1ull));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) ddr_bar_addr = goya_set_ddr_bar_base(hdev, bar_base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) if (ddr_bar_addr != U64_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) (addr - bar_base_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) ddr_bar_addr = goya_set_ddr_bar_base(hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) ddr_bar_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) if (ddr_bar_addr == U64_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) rc = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) } else if (addr >= HOST_PHYS_BASE && !iommu_present(&pci_bus_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) *(u64 *) phys_to_virt(addr - HOST_PHYS_BASE) = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) static u64 goya_read_pte(struct hl_device *hdev, u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) if (hdev->hard_reset_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) return U64_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) return readq(hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) (addr - goya->ddr_bar_cur_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) static void goya_write_pte(struct hl_device *hdev, u64 addr, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) if (hdev->hard_reset_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) writeq(val, hdev->pcie_bar[DDR_BAR_ID] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) (addr - goya->ddr_bar_cur_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) static const char *_goya_get_event_desc(u16 event_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) switch (event_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) case GOYA_ASYNC_EVENT_ID_PCIE_IF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) return "PCIe_if";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) return "TPC%d_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) case GOYA_ASYNC_EVENT_ID_MME_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) return "MME_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) return "MME_ecc_ext";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) case GOYA_ASYNC_EVENT_ID_MMU_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) return "MMU_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) return "DMA_macro";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) case GOYA_ASYNC_EVENT_ID_DMA_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) return "DMA_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) return "CPU_if_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) return "PSOC_mem";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) return "PSOC_coresight";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) return "SRAM%d";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) case GOYA_ASYNC_EVENT_ID_GIC500:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) return "GIC500";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) return "PLL%d";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) case GOYA_ASYNC_EVENT_ID_AXI_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) return "AXI_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) return "L2_ram_ecc";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) return "PSOC_gpio_05_sw_reset";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) return "PSOC_gpio_10_vrhot_icrit";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) return "PCIe_dec";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) return "TPC%d_dec";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) case GOYA_ASYNC_EVENT_ID_MME_WACS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) return "MME_wacs";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) case GOYA_ASYNC_EVENT_ID_MME_WACSD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) return "MME_wacsd";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) return "CPU_axi_splitter";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) return "PSOC_axi_dec";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) case GOYA_ASYNC_EVENT_ID_PSOC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) return "PSOC";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) return "TPC%d_krn_err";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) return "TPC%d_cq";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) return "TPC%d_qm";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) case GOYA_ASYNC_EVENT_ID_MME_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) return "MME_qm";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) return "MME_cq";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) return "DMA%d_qm";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) return "DMA%d_ch";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) return "TPC%d_bmon_spmu";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) return "DMA_bm_ch%d";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) return "POWER_ENV_S";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) return "POWER_ENV_E";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) return "THERMAL_ENV_S";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) return "THERMAL_ENV_E";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) return "N/A";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) static void goya_get_event_desc(u16 event_type, char *desc, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) u8 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) switch (event_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_ECC) / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) index = event_type - GOYA_ASYNC_EVENT_ID_SRAM0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) index = event_type - GOYA_ASYNC_EVENT_ID_PLL0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_DEC) / 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR) / 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_CMDQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_CMDQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) case GOYA_ASYNC_EVENT_ID_TPC0_QM ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) index = event_type - GOYA_ASYNC_EVENT_ID_TPC0_QM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_QM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) index = event_type - GOYA_ASYNC_EVENT_ID_DMA0_CH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) index = (event_type - GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU) / 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) index = event_type - GOYA_ASYNC_EVENT_ID_DMA_BM_CH0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) snprintf(desc, size, _goya_get_event_desc(event_type), index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) snprintf(desc, size, _goya_get_event_desc(event_type));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) static void goya_print_razwi_info(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) if (RREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) dev_err_ratelimited(hdev->dev, "Illegal write to LBW\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) WREG32(mmDMA_MACRO_RAZWI_LBW_WT_VLD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) if (RREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) dev_err_ratelimited(hdev->dev, "Illegal read from LBW\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) WREG32(mmDMA_MACRO_RAZWI_LBW_RD_VLD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) if (RREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) dev_err_ratelimited(hdev->dev, "Illegal write to HBW\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) WREG32(mmDMA_MACRO_RAZWI_HBW_WT_VLD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) if (RREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) dev_err_ratelimited(hdev->dev, "Illegal read from HBW\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) WREG32(mmDMA_MACRO_RAZWI_HBW_RD_VLD, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) static void goya_print_mmu_error_info(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) val = RREG32(mmMMU_PAGE_ERROR_CAPTURE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) if (val & MMU_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) addr = val & MMU_PAGE_ERROR_CAPTURE_VA_49_32_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) addr <<= 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) addr |= RREG32(mmMMU_PAGE_ERROR_CAPTURE_VA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) WREG32(mmMMU_PAGE_ERROR_CAPTURE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) static void goya_print_irq_info(struct hl_device *hdev, u16 event_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) bool razwi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) char desc[20] = "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) goya_get_event_desc(event_type, desc, sizeof(desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) event_type, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) if (razwi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) goya_print_razwi_info(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) goya_print_mmu_error_info(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) size_t irq_arr_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) struct cpucp_unmask_irq_arr_packet *pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) size_t total_pkt_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) long result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) int irq_num_entries, irq_arr_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) __le32 *goya_irq_arr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) irq_arr_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) /* data should be aligned to 8 bytes in order to CPU-CP to copy it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) total_pkt_size = (total_pkt_size + 0x7) & ~0x7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) /* total_pkt_size is casted to u16 later on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) if (total_pkt_size > USHRT_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) dev_err(hdev->dev, "too many elements in IRQ array\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) pkt = kzalloc(total_pkt_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) if (!pkt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) irq_num_entries = irq_arr_size / sizeof(irq_arr[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) pkt->length = cpu_to_le32(irq_num_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) /* We must perform any necessary endianness conversation on the irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) * array being passed to the goya hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) for (irq_arr_index = 0, goya_irq_arr = (__le32 *) &pkt->irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) irq_arr_index < irq_num_entries ; irq_arr_index++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) goya_irq_arr[irq_arr_index] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) cpu_to_le32(irq_arr[irq_arr_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) CPUCP_PKT_CTL_OPCODE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) total_pkt_size, 0, &result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) dev_err(hdev->dev, "failed to unmask IRQ array\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) kfree(pkt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) static int goya_soft_reset_late_init(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) * Unmask all IRQs since some could have been received
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) * during the soft reset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) return goya_unmask_irq_arr(hdev, goya_all_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) sizeof(goya_all_events));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) static int goya_unmask_irq(struct hl_device *hdev, u16 event_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) struct cpucp_packet pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) long result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) memset(&pkt, 0, sizeof(pkt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) CPUCP_PKT_CTL_OPCODE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) pkt.value = cpu_to_le64(event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) 0, &result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) dev_err(hdev->dev, "failed to unmask RAZWI IRQ %d", event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) switch (event_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) dev_info_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) "Clock throttling due to power consumption\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) dev_info_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) "Power envelop is safe, back to optimal clock\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) dev_info_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) "Clock throttling due to overheating\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) dev_info_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) "Thermal envelop is safe, back to optimal clock\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) dev_err(hdev->dev, "Received invalid clock change event %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) >> EQ_CTL_EVENT_TYPE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) if (event_type >= GOYA_ASYNC_EVENT_ID_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) dev_err(hdev->dev, "Event type %u exceeds maximum of %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) event_type, GOYA_ASYNC_EVENT_ID_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) goya->events_stat[event_type]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) goya->events_stat_aggregate[event_type]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) switch (event_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) case GOYA_ASYNC_EVENT_ID_PCIE_IF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) case GOYA_ASYNC_EVENT_ID_TPC0_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) case GOYA_ASYNC_EVENT_ID_TPC1_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) case GOYA_ASYNC_EVENT_ID_TPC2_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) case GOYA_ASYNC_EVENT_ID_TPC3_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) case GOYA_ASYNC_EVENT_ID_TPC4_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) case GOYA_ASYNC_EVENT_ID_TPC5_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) case GOYA_ASYNC_EVENT_ID_TPC6_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) case GOYA_ASYNC_EVENT_ID_TPC7_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) case GOYA_ASYNC_EVENT_ID_MME_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) case GOYA_ASYNC_EVENT_ID_MME_ECC_EXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) case GOYA_ASYNC_EVENT_ID_MMU_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) case GOYA_ASYNC_EVENT_ID_DMA_MACRO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) case GOYA_ASYNC_EVENT_ID_DMA_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) case GOYA_ASYNC_EVENT_ID_CPU_IF_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) case GOYA_ASYNC_EVENT_ID_PSOC_MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) case GOYA_ASYNC_EVENT_ID_PSOC_CORESIGHT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) case GOYA_ASYNC_EVENT_ID_SRAM0 ... GOYA_ASYNC_EVENT_ID_SRAM29:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) case GOYA_ASYNC_EVENT_ID_GIC500:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) case GOYA_ASYNC_EVENT_ID_PLL0 ... GOYA_ASYNC_EVENT_ID_PLL6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) case GOYA_ASYNC_EVENT_ID_AXI_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) case GOYA_ASYNC_EVENT_ID_L2_RAM_ECC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_05_SW_RESET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) goya_print_irq_info(hdev, event_type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) if (hdev->hard_reset_on_fw_events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) hl_device_reset(hdev, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) case GOYA_ASYNC_EVENT_ID_PCIE_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) case GOYA_ASYNC_EVENT_ID_TPC0_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) case GOYA_ASYNC_EVENT_ID_TPC1_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) case GOYA_ASYNC_EVENT_ID_TPC2_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) case GOYA_ASYNC_EVENT_ID_TPC3_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) case GOYA_ASYNC_EVENT_ID_TPC4_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) case GOYA_ASYNC_EVENT_ID_TPC5_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) case GOYA_ASYNC_EVENT_ID_TPC6_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) case GOYA_ASYNC_EVENT_ID_TPC7_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) case GOYA_ASYNC_EVENT_ID_MME_WACS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) case GOYA_ASYNC_EVENT_ID_MME_WACSD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) case GOYA_ASYNC_EVENT_ID_CPU_AXI_SPLITTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) case GOYA_ASYNC_EVENT_ID_PSOC_AXI_DEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) case GOYA_ASYNC_EVENT_ID_PSOC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) case GOYA_ASYNC_EVENT_ID_TPC0_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) case GOYA_ASYNC_EVENT_ID_TPC1_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) case GOYA_ASYNC_EVENT_ID_TPC2_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) case GOYA_ASYNC_EVENT_ID_TPC3_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) case GOYA_ASYNC_EVENT_ID_TPC4_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) case GOYA_ASYNC_EVENT_ID_TPC5_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) case GOYA_ASYNC_EVENT_ID_TPC6_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) case GOYA_ASYNC_EVENT_ID_TPC7_KRN_ERR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) case GOYA_ASYNC_EVENT_ID_TPC0_CMDQ ... GOYA_ASYNC_EVENT_ID_TPC7_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) case GOYA_ASYNC_EVENT_ID_MME_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) case GOYA_ASYNC_EVENT_ID_MME_CMDQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) case GOYA_ASYNC_EVENT_ID_DMA0_QM ... GOYA_ASYNC_EVENT_ID_DMA4_QM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) case GOYA_ASYNC_EVENT_ID_DMA0_CH ... GOYA_ASYNC_EVENT_ID_DMA4_CH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) goya_print_irq_info(hdev, event_type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) goya_unmask_irq(hdev, event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) case GOYA_ASYNC_EVENT_ID_PSOC_GPIO_10_VRHOT_ICRIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) case GOYA_ASYNC_EVENT_ID_TPC0_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) case GOYA_ASYNC_EVENT_ID_TPC1_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) case GOYA_ASYNC_EVENT_ID_TPC2_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699) case GOYA_ASYNC_EVENT_ID_TPC3_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) case GOYA_ASYNC_EVENT_ID_TPC4_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) case GOYA_ASYNC_EVENT_ID_TPC5_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) case GOYA_ASYNC_EVENT_ID_TPC6_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) case GOYA_ASYNC_EVENT_ID_TPC7_BMON_SPMU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) case GOYA_ASYNC_EVENT_ID_DMA_BM_CH0 ... GOYA_ASYNC_EVENT_ID_DMA_BM_CH4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) goya_print_irq_info(hdev, event_type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) goya_unmask_irq(hdev, event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) goya_print_clk_change_info(hdev, event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) goya_unmask_irq(hdev, event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) event_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) void *goya_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) if (aggregate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) *size = (u32) sizeof(goya->events_stat_aggregate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) return goya->events_stat_aggregate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) *size = (u32) sizeof(goya->events_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) return goya->events_stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) static int goya_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) u64 val, bool is_dram)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) struct packet_lin_dma *lin_dma_pkt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) struct hl_cs_job *job;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) u32 cb_size, ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) struct hl_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) int rc, lin_dma_pkts_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) lin_dma_pkts_cnt = DIV_ROUND_UP_ULL(size, SZ_2G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) cb_size = lin_dma_pkts_cnt * sizeof(struct packet_lin_dma) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) sizeof(struct packet_msg_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) cb = hl_cb_kernel_create(hdev, cb_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) if (!cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) lin_dma_pkt = cb->kernel_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) ctl = ((PACKET_LIN_DMA << GOYA_PKT_CTL_OPCODE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) (1 << GOYA_PKT_LIN_DMA_CTL_MEMSET_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) (1 << GOYA_PKT_LIN_DMA_CTL_WO_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) (1 << GOYA_PKT_CTL_RB_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) (1 << GOYA_PKT_CTL_MB_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) ctl |= (is_dram ? DMA_HOST_TO_DRAM : DMA_HOST_TO_SRAM) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) GOYA_PKT_LIN_DMA_CTL_DMA_DIR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) lin_dma_pkt->ctl = cpu_to_le32(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) lin_dma_pkt->src_addr = cpu_to_le64(val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) lin_dma_pkt->dst_addr = cpu_to_le64(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) if (lin_dma_pkts_cnt > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) lin_dma_pkt->tsize = cpu_to_le32(SZ_2G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) lin_dma_pkt->tsize = cpu_to_le32(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) size -= SZ_2G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) addr += SZ_2G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) lin_dma_pkt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) } while (--lin_dma_pkts_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) if (!job) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) dev_err(hdev->dev, "Failed to allocate a new job\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) goto release_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) job->id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) job->user_cb = cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) job->user_cb->cs_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) job->user_cb_size = cb_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) job->hw_queue_id = GOYA_QUEUE_ID_DMA_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) job->patched_cb = job->user_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) job->job_cb_size = job->user_cb_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) hl_debugfs_add_job(hdev, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) rc = goya_send_job_on_qman0(hdev, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) hl_debugfs_remove_job(hdev, job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) kfree(job);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) cb->cs_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) release_cb:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803) hl_cb_put(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) int goya_context_switch(struct hl_device *hdev, u32 asid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) u64 addr = prop->sram_base_address, sob_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) u32 size = hdev->pldm ? 0x10000 : prop->sram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) u64 val = 0x7777777777777777ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) int rc, dma_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816) u32 channel_off = mmDMA_CH_1_WR_COMP_ADDR_LO -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) mmDMA_CH_0_WR_COMP_ADDR_LO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) rc = goya_memset_device_memory(hdev, addr, size, val, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) dev_err(hdev->dev, "Failed to clear SRAM in context switch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) /* we need to reset registers that the user is allowed to change */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1007;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO, lower_32_bits(sob_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) for (dma_id = 1 ; dma_id < NUMBER_OF_EXT_HW_QUEUES ; dma_id++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) sob_addr = CFG_BASE + mmSYNC_MNGR_SOB_OBJ_1000 +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) (dma_id - 1) * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) WREG32(mmDMA_CH_0_WR_COMP_ADDR_LO + channel_off * dma_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) lower_32_bits(sob_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) WREG32(mmTPC_PLL_CLK_RLX_0, 0x200020);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) goya_mmu_prepare(hdev, asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) goya_clear_sm_regs(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) static int goya_mmu_clear_pgt_range(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) u64 addr = prop->mmu_pgt_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) u32 size = prop->mmu_pgt_size + MMU_DRAM_DEFAULT_PAGE_SIZE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) MMU_CACHE_MNG_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) return goya_memset_device_memory(hdev, addr, size, 0, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) static int goya_mmu_set_dram_default_page(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) u64 addr = hdev->asic_prop.mmu_dram_default_page_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) u32 size = MMU_DRAM_DEFAULT_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) u64 val = 0x9999999999999999ull;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) return goya_memset_device_memory(hdev, addr, size, val, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) s64 off, cpu_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) rc = hl_mmu_map(hdev->kernel_ctx, prop->dram_base_address + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) prop->dram_base_address + off, PAGE_SIZE_2MB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) (off + PAGE_SIZE_2MB) == CPU_FW_IMAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) dev_err(hdev->dev, "Map failed for address 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) prop->dram_base_address + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) rc = hl_mmu_map(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) hdev->cpu_accessible_dma_address, PAGE_SIZE_2MB, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) "Map failed for CPU accessible memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) off -= PAGE_SIZE_2MB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) goto unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) rc = hl_mmu_map(hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) hdev->cpu_accessible_dma_address + cpu_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) PAGE_SIZE_4KB, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) "Map failed for CPU accessible memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) cpu_off -= PAGE_SIZE_4KB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) goto unmap_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) goya_mmu_prepare_reg(hdev, mmCPU_IF_ARUSER_OVR, HL_KERNEL_ASID_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) goya_mmu_prepare_reg(hdev, mmCPU_IF_AWUSER_OVR, HL_KERNEL_ASID_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) WREG32(mmCPU_IF_ARUSER_OVR_EN, 0x7FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) WREG32(mmCPU_IF_AWUSER_OVR_EN, 0x7FF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) /* Make sure configuration is flushed to device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924) RREG32(mmCPU_IF_AWUSER_OVR_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) goya->device_cpu_mmu_mappings_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) unmap_cpu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931) for (; cpu_off >= 0 ; cpu_off -= PAGE_SIZE_4KB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) if (hl_mmu_unmap(hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) PAGE_SIZE_4KB, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) dev_warn_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) "failed to unmap address 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) for (; off >= 0 ; off -= PAGE_SIZE_2MB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) if (hl_mmu_unmap(hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) prop->dram_base_address + off, PAGE_SIZE_2MB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) dev_warn_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) "failed to unmap address 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) prop->dram_base_address + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) void goya_mmu_remove_device_cpu_mappings(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) u32 off, cpu_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) if (!goya->device_cpu_mmu_mappings_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) WREG32(mmCPU_IF_ARUSER_OVR_EN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) WREG32(mmCPU_IF_AWUSER_OVR_EN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) if (!(hdev->cpu_accessible_dma_address & (PAGE_SIZE_2MB - 1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) if (hl_mmu_unmap(hdev->kernel_ctx, VA_CPU_ACCESSIBLE_MEM_ADDR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) PAGE_SIZE_2MB, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) dev_warn(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) "Failed to unmap CPU accessible memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) for (cpu_off = 0 ; cpu_off < SZ_2M ; cpu_off += PAGE_SIZE_4KB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) if (hl_mmu_unmap(hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) PAGE_SIZE_4KB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) (cpu_off + PAGE_SIZE_4KB) >= SZ_2M))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) dev_warn_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) "failed to unmap address 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) VA_CPU_ACCESSIBLE_MEM_ADDR + cpu_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) for (off = 0 ; off < CPU_FW_IMAGE_SIZE ; off += PAGE_SIZE_2MB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) if (hl_mmu_unmap(hdev->kernel_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) prop->dram_base_address + off, PAGE_SIZE_2MB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) (off + PAGE_SIZE_2MB) >= CPU_FW_IMAGE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) dev_warn_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) "Failed to unmap address 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) prop->dram_base_address + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) goya->device_cpu_mmu_mappings_done = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) static void goya_mmu_prepare(struct hl_device *hdev, u32 asid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) if (!(goya->hw_cap_initialized & HW_CAP_MMU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) if (asid & ~MME_QM_GLBL_SECURE_PROPS_ASID_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) WARN(1, "asid %u is too big\n", asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) /* zero the MMBP and ASID bits and then set the ASID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) for (i = 0 ; i < GOYA_MMU_REGS_NUM ; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) goya_mmu_prepare_reg(hdev, goya_mmu_regs[i], asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) u32 status, timeout_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) hdev->hard_reset_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) /* no need in L1 only invalidation in Goya */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) if (!is_hard)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) mutex_lock(&hdev->mmu_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) /* L0 & L1 invalidation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) WREG32(mmSTLB_INV_ALL_START, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) rc = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) mmSTLB_INV_ALL_START,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) !status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) timeout_usec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) mutex_unlock(&hdev->mmu_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) dev_err_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) "MMU cache invalidation timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) hl_device_reset(hdev, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) static int goya_mmu_invalidate_cache_range(struct hl_device *hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) bool is_hard, u32 asid, u64 va, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) u32 status, timeout_usec, inv_data, pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) hdev->hard_reset_pending)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) /* no need in L1 only invalidation in Goya */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) if (!is_hard)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) if (hdev->pldm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) mutex_lock(&hdev->mmu_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) * TODO: currently invalidate entire L0 & L1 as in regular hard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) * invalidation. Need to apply invalidation of specific cache lines with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) * mask of ASID & VA & size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080) * Note that L1 with be flushed entirely in any case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) /* L0 & L1 invalidation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) inv_data = RREG32(mmSTLB_CACHE_INV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) /* PI is 8 bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) WREG32(mmSTLB_CACHE_INV,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090) rc = hl_poll_timeout(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) hdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) mmSTLB_INV_CONSUMER_INDEX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) status == pi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) 1000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) timeout_usec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) mutex_unlock(&hdev->mmu_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) dev_err_ratelimited(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) "MMU cache invalidation timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) hl_device_reset(hdev, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) int goya_send_heartbeat(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) return hl_fw_send_heartbeat(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) int goya_cpucp_info_get(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) struct asic_fixed_properties *prop = &hdev->asic_prop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) u64 dram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129) rc = hl_fw_cpucp_info_get(hdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) dram_size = le64_to_cpu(prop->cpucp_info.dram_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) if (dram_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) if ((!is_power_of_2(dram_size)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) (dram_size < DRAM_PHYS_DEFAULT_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) dev_err(hdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) "F/W reported invalid DRAM size %llu. Trying to use default size\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) dram_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) dram_size = DRAM_PHYS_DEFAULT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) prop->dram_size = dram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) prop->dram_end_address = prop->dram_base_address + dram_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147) if (!strlen(prop->cpucp_info.card_name))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) strncpy(prop->cpucp_info.card_name, GOYA_DEFAULT_CARD_NAME,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) CARD_NAME_MAX_LEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) static void goya_set_clock_gating(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) /* clock gating not supported in Goya */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) static void goya_disable_clock_gating(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) /* clock gating not supported in Goya */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) struct seq_file *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) const char *dma_fmt = "%-5d%-9s%#-14x%#x\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) u32 qm_glbl_sts0, cmdq_glbl_sts0, dma_core_sts0, tpc_cfg_sts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) mme_arch_sts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) bool is_idle = true, is_eng_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) u64 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) if (s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) seq_puts(s, "\nDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) "--- ------- ------------ -------------\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) for (i = 0 ; i < DMA_MAX_NUM ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) qm_glbl_sts0 = RREG32(mmDMA_QM_0_GLBL_STS0 + i * offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) dma_core_sts0 = RREG32(mmDMA_CH_0_STS0 + i * offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) is_eng_idle = IS_DMA_QM_IDLE(qm_glbl_sts0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) IS_DMA_IDLE(dma_core_sts0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) is_idle &= is_eng_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) if (mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) *mask |= ((u64) !is_eng_idle) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) (GOYA_ENGINE_ID_DMA_0 + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) if (s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) qm_glbl_sts0, dma_core_sts0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) if (s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) seq_puts(s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) "\nTPC is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 CFG_STATUS\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) "--- ------- ------------ -------------- ----------\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) offset = mmTPC1_QM_GLBL_STS0 - mmTPC0_QM_GLBL_STS0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) for (i = 0 ; i < TPC_MAX_NUM ; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + i * offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) cmdq_glbl_sts0 = RREG32(mmTPC0_CMDQ_GLBL_STS0 + i * offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206) tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + i * offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) is_eng_idle = IS_TPC_QM_IDLE(qm_glbl_sts0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) IS_TPC_CMDQ_IDLE(cmdq_glbl_sts0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) IS_TPC_IDLE(tpc_cfg_sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) is_idle &= is_eng_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) if (mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) *mask |= ((u64) !is_eng_idle) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214) (GOYA_ENGINE_ID_TPC_0 + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) if (s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) if (s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) seq_puts(s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) "\nMME is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 ARCH_STATUS\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) "--- ------- ------------ -------------- -----------\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) qm_glbl_sts0 = RREG32(mmMME_QM_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226) cmdq_glbl_sts0 = RREG32(mmMME_CMDQ_GLBL_STS0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) mme_arch_sts = RREG32(mmMME_ARCH_STATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) is_eng_idle = IS_MME_QM_IDLE(qm_glbl_sts0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) IS_MME_CMDQ_IDLE(cmdq_glbl_sts0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) IS_MME_IDLE(mme_arch_sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) is_idle &= is_eng_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) if (mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) *mask |= ((u64) !is_eng_idle) << GOYA_ENGINE_ID_MME_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) if (s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237) cmdq_glbl_sts0, mme_arch_sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) seq_puts(s, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) return is_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244) static void goya_hw_queues_lock(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) __acquires(&goya->hw_queues_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) spin_lock(&goya->hw_queues_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) static void goya_hw_queues_unlock(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) __releases(&goya->hw_queues_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) spin_unlock(&goya->hw_queues_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) static u32 goya_get_pci_id(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) return hdev->pdev->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) size_t max_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) struct goya_device *goya = hdev->asic_specific;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) return hl_fw_get_eeprom_data(hdev, data, max_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) static enum hl_device_hw_state goya_get_hw_state(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) return RREG32(mmHW_STATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) static int goya_ctx_init(struct hl_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) u32 goya_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) return cq_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) static u32 goya_get_signal_cb_size(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) static u32 goya_get_wait_cb_size(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) static void goya_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) static void goya_gen_wait_cb(struct hl_device *hdev, void *data, u16 sob_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) u16 sob_val, u16 mon_id, u32 q_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) static void goya_reset_sob(struct hl_device *hdev, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) static void goya_set_dma_mask_from_fw(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) if (RREG32(mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) HL_POWER9_HOST_MAGIC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) dev_dbg(hdev->dev, "Working in 64-bit DMA mode\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) hdev->power9_64bit_dma_enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) hdev->dma_mask = 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) dev_dbg(hdev->dev, "Working in 48-bit DMA mode\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) hdev->power9_64bit_dma_enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) hdev->dma_mask = 48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) u64 goya_get_device_time(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) static const struct hl_asic_funcs goya_funcs = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) .early_init = goya_early_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) .early_fini = goya_early_fini,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) .late_init = goya_late_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) .late_fini = goya_late_fini,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) .sw_init = goya_sw_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) .sw_fini = goya_sw_fini,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) .hw_init = goya_hw_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) .hw_fini = goya_hw_fini,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) .halt_engines = goya_halt_engines,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) .suspend = goya_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) .resume = goya_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) .cb_mmap = goya_cb_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) .ring_doorbell = goya_ring_doorbell,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) .pqe_write = goya_pqe_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) .asic_dma_alloc_coherent = goya_dma_alloc_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) .asic_dma_free_coherent = goya_dma_free_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355) .get_int_queue_base = goya_get_int_queue_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) .test_queues = goya_test_queues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357) .asic_dma_pool_zalloc = goya_dma_pool_zalloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) .asic_dma_pool_free = goya_dma_pool_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) .cpu_accessible_dma_pool_alloc = goya_cpu_accessible_dma_pool_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) .cpu_accessible_dma_pool_free = goya_cpu_accessible_dma_pool_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) .hl_dma_unmap_sg = goya_dma_unmap_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) .cs_parser = goya_cs_parser,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) .asic_dma_map_sg = goya_dma_map_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) .get_dma_desc_list_size = goya_get_dma_desc_list_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) .add_end_of_cb_packets = goya_add_end_of_cb_packets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) .update_eq_ci = goya_update_eq_ci,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) .context_switch = goya_context_switch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) .restore_phase_topology = goya_restore_phase_topology,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) .debugfs_read32 = goya_debugfs_read32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) .debugfs_write32 = goya_debugfs_write32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) .debugfs_read64 = goya_debugfs_read64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) .debugfs_write64 = goya_debugfs_write64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373) .add_device_attr = goya_add_device_attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) .handle_eqe = goya_handle_eqe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) .set_pll_profile = goya_set_pll_profile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376) .get_events_stat = goya_get_events_stat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) .read_pte = goya_read_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378) .write_pte = goya_write_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) .mmu_invalidate_cache = goya_mmu_invalidate_cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) .mmu_invalidate_cache_range = goya_mmu_invalidate_cache_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) .send_heartbeat = goya_send_heartbeat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) .set_clock_gating = goya_set_clock_gating,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) .disable_clock_gating = goya_disable_clock_gating,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384) .debug_coresight = goya_debug_coresight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) .is_device_idle = goya_is_device_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) .soft_reset_late_init = goya_soft_reset_late_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387) .hw_queues_lock = goya_hw_queues_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) .hw_queues_unlock = goya_hw_queues_unlock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389) .get_pci_id = goya_get_pci_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) .get_eeprom_data = goya_get_eeprom_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) .send_cpu_message = goya_send_cpu_message,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392) .get_hw_state = goya_get_hw_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) .pci_bars_map = goya_pci_bars_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) .init_iatu = goya_init_iatu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) .rreg = hl_rreg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396) .wreg = hl_wreg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) .halt_coresight = goya_halt_coresight,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) .ctx_init = goya_ctx_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) .get_clk_rate = goya_get_clk_rate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400) .get_queue_id_for_cq = goya_get_queue_id_for_cq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) .read_device_fw_version = goya_read_device_fw_version,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) .load_firmware_to_device = goya_load_firmware_to_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403) .load_boot_fit_to_device = goya_load_boot_fit_to_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) .get_signal_cb_size = goya_get_signal_cb_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) .get_wait_cb_size = goya_get_wait_cb_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) .gen_signal_cb = goya_gen_signal_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407) .gen_wait_cb = goya_gen_wait_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) .reset_sob = goya_reset_sob,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) .set_dma_mask_from_fw = goya_set_dma_mask_from_fw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) .get_device_time = goya_get_device_time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) * goya_set_asic_funcs - set Goya function pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) * @*hdev: pointer to hl_device structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) void goya_set_asic_funcs(struct hl_device *hdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) hdev->asic_funcs = &goya_funcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags