^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * IOMMU API for ARM architected SMMUv3 implementations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2015 ARM Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #ifndef _ARM_SMMU_V3_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #define _ARM_SMMU_V3_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bitfield.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/mmzone.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) /* MMIO registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define ARM_SMMU_IDR0 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define IDR0_ST_LVL GENMASK(28, 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define IDR0_ST_LVL_2LVL 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define IDR0_STALL_MODEL GENMASK(25, 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define IDR0_STALL_MODEL_STALL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define IDR0_STALL_MODEL_FORCE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define IDR0_TTENDIAN GENMASK(22, 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define IDR0_TTENDIAN_MIXED 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define IDR0_TTENDIAN_LE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define IDR0_TTENDIAN_BE 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define IDR0_CD2L (1 << 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define IDR0_VMID16 (1 << 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define IDR0_PRI (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define IDR0_SEV (1 << 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define IDR0_MSI (1 << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define IDR0_ASID16 (1 << 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define IDR0_ATS (1 << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define IDR0_HYP (1 << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define IDR0_COHACC (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define IDR0_TTF GENMASK(3, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define IDR0_TTF_AARCH64 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define IDR0_TTF_AARCH32_64 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define IDR0_S1P (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define IDR0_S2P (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define ARM_SMMU_IDR1 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define IDR1_TABLES_PRESET (1 << 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define IDR1_QUEUES_PRESET (1 << 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define IDR1_REL (1 << 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define IDR1_CMDQS GENMASK(25, 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define IDR1_EVTQS GENMASK(20, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define IDR1_PRIQS GENMASK(15, 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define IDR1_SSIDSIZE GENMASK(10, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define IDR1_SIDSIZE GENMASK(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define ARM_SMMU_IDR3 0xc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define IDR3_RIL (1 << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define ARM_SMMU_IDR5 0x14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define IDR5_STALL_MAX GENMASK(31, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define IDR5_GRAN64K (1 << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define IDR5_GRAN16K (1 << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define IDR5_GRAN4K (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define IDR5_OAS GENMASK(2, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define IDR5_OAS_32_BIT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define IDR5_OAS_36_BIT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define IDR5_OAS_40_BIT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define IDR5_OAS_42_BIT 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define IDR5_OAS_44_BIT 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define IDR5_OAS_48_BIT 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define IDR5_OAS_52_BIT 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define IDR5_VAX GENMASK(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define IDR5_VAX_52_BIT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define ARM_SMMU_CR0 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define CR0_ATSCHK (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define CR0_CMDQEN (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define CR0_EVTQEN (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define CR0_PRIQEN (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define CR0_SMMUEN (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define ARM_SMMU_CR0ACK 0x24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define ARM_SMMU_CR1 0x28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define CR1_TABLE_SH GENMASK(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define CR1_TABLE_OC GENMASK(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define CR1_TABLE_IC GENMASK(7, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define CR1_QUEUE_SH GENMASK(5, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define CR1_QUEUE_OC GENMASK(3, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define CR1_QUEUE_IC GENMASK(1, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define CR1_CACHE_NC 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define CR1_CACHE_WB 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define CR1_CACHE_WT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define ARM_SMMU_CR2 0x2c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define CR2_PTM (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define CR2_RECINVSID (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define CR2_E2H (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define ARM_SMMU_GBPA 0x44
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define GBPA_UPDATE (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define GBPA_ABORT (1 << 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define ARM_SMMU_IRQ_CTRL 0x50
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #define ARM_SMMU_IRQ_CTRLACK 0x54
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define ARM_SMMU_GERROR 0x60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define GERROR_SFM_ERR (1 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define GERROR_PRIQ_ABT_ERR (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define GERROR_EVTQ_ABT_ERR (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define GERROR_CMDQ_ERR (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #define GERROR_ERR_MASK 0x1fd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define ARM_SMMU_GERRORN 0x64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define ARM_SMMU_STRTAB_BASE 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define STRTAB_BASE_RA (1UL << 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define ARM_SMMU_STRTAB_BASE_CFG 0x88
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define STRTAB_BASE_CFG_FMT_LINEAR 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define STRTAB_BASE_CFG_FMT_2LVL 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define ARM_SMMU_CMDQ_BASE 0x90
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define ARM_SMMU_CMDQ_PROD 0x98
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define ARM_SMMU_CMDQ_CONS 0x9c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define ARM_SMMU_EVTQ_BASE 0xa0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define ARM_SMMU_EVTQ_PROD 0x100a8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define ARM_SMMU_EVTQ_CONS 0x100ac
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define ARM_SMMU_PRIQ_BASE 0xc0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define ARM_SMMU_PRIQ_PROD 0x100c8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define ARM_SMMU_PRIQ_CONS 0x100cc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define ARM_SMMU_REG_SZ 0xe00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /* Common MSI config fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define MSI_CFG2_SH GENMASK(5, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define MSI_CFG2_MEMATTR GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /* Common memory attribute values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define ARM_SMMU_SH_NSH 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #define ARM_SMMU_SH_OSH 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define ARM_SMMU_SH_ISH 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define ARM_SMMU_MEMATTR_OIWB 0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #define Q_OVERFLOW_FLAG (1U << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #define Q_ENT(q, p) ((q)->base + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) Q_IDX(&((q)->llq), p) * \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) (q)->ent_dwords)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define Q_BASE_RWA (1UL << 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #define Q_BASE_LOG2SIZE GENMASK(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /* Ensure DMA allocations are naturally aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #ifdef CONFIG_CMA_ALIGNMENT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Stream table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * 2lvl: 128k L1 entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * 256 lazy entries per table (each table covers a PCI bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) #define STRTAB_L1_SZ_SHIFT 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #define STRTAB_SPLIT 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define STRTAB_L1_DESC_DWORDS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define STRTAB_STE_DWORDS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define STRTAB_STE_0_V (1UL << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #define STRTAB_STE_0_CFG_ABORT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #define STRTAB_STE_0_CFG_BYPASS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #define STRTAB_STE_0_CFG_S1_TRANS 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #define STRTAB_STE_0_CFG_S2_TRANS 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #define STRTAB_STE_0_S1FMT_LINEAR 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #define STRTAB_STE_0_S1FMT_64K_L2 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #define STRTAB_STE_1_S1DSS GENMASK_ULL(1, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #define STRTAB_STE_1_S1DSS_TERMINATE 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) #define STRTAB_STE_1_S1DSS_BYPASS 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #define STRTAB_STE_1_S1DSS_SSID0 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) #define STRTAB_STE_1_S1C_CACHE_NC 0UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #define STRTAB_STE_1_S1C_CACHE_WT 2UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #define STRTAB_STE_1_S1C_CACHE_WB 3UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define STRTAB_STE_1_S1STALLD (1UL << 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #define STRTAB_STE_1_EATS_ABT 0UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #define STRTAB_STE_1_EATS_TRANS 1UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #define STRTAB_STE_1_EATS_S1CHK 2UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) #define STRTAB_STE_1_STRW_NSEL1 0UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) #define STRTAB_STE_1_STRW_EL2 2UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) #define STRTAB_STE_1_SHCFG_INCOMING 1UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) #define STRTAB_STE_2_VTCR_S2T0SZ GENMASK_ULL(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #define STRTAB_STE_2_VTCR_S2SL0 GENMASK_ULL(7, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #define STRTAB_STE_2_VTCR_S2IR0 GENMASK_ULL(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #define STRTAB_STE_2_VTCR_S2OR0 GENMASK_ULL(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #define STRTAB_STE_2_VTCR_S2SH0 GENMASK_ULL(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #define STRTAB_STE_2_VTCR_S2TG GENMASK_ULL(15, 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #define STRTAB_STE_2_VTCR_S2PS GENMASK_ULL(18, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #define STRTAB_STE_2_S2AA64 (1UL << 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #define STRTAB_STE_2_S2ENDI (1UL << 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #define STRTAB_STE_2_S2PTW (1UL << 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) #define STRTAB_STE_2_S2R (1UL << 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * Context descriptors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * Linear: when less than 1024 SSIDs are supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * 2lvl: at most 1024 L1 entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * 1024 lazy entries per table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #define CTXDESC_SPLIT 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) #define CTXDESC_L2_ENTRIES (1 << CTXDESC_SPLIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) #define CTXDESC_L1_DESC_DWORDS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) #define CTXDESC_L1_DESC_V (1UL << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) #define CTXDESC_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) #define CTXDESC_CD_DWORDS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #define CTXDESC_CD_0_ENDI (1UL << 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #define CTXDESC_CD_0_V (1UL << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #define CTXDESC_CD_0_AA64 (1UL << 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #define CTXDESC_CD_0_S (1UL << 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #define CTXDESC_CD_0_R (1UL << 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #define CTXDESC_CD_0_A (1UL << 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #define CTXDESC_CD_0_ASET (1UL << 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * When the SMMU only supports linear context descriptor tables, pick a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * reasonable size limit (64kB).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #define CTXDESC_LINEAR_CDMAX ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* Command queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #define CMDQ_ENT_SZ_SHIFT 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #define CMDQ_CONS_ERR GENMASK(30, 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) #define CMDQ_ERR_CERROR_NONE_IDX 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #define CMDQ_ERR_CERROR_ILL_IDX 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #define CMDQ_ERR_CERROR_ABT_IDX 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * This is used to size the command queue and therefore must be at least
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * BITS_PER_LONG so that the valid_map works correctly (it relies on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * total number of queue entries being a multiple of BITS_PER_LONG).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) #define CMDQ_0_OP GENMASK_ULL(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #define CMDQ_0_SSV (1UL << 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #define CMDQ_CFGI_0_SSID GENMASK_ULL(31, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #define CMDQ_CFGI_1_LEAF (1UL << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) #define CMDQ_TLBI_0_NUM GENMASK_ULL(16, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #define CMDQ_TLBI_RANGE_NUM_MAX 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) #define CMDQ_TLBI_0_SCALE GENMASK_ULL(24, 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #define CMDQ_TLBI_1_LEAF (1UL << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) #define CMDQ_TLBI_1_TTL GENMASK_ULL(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) #define CMDQ_TLBI_1_TG GENMASK_ULL(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define CMDQ_ATC_0_GLOBAL (1UL << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) #define CMDQ_SYNC_0_CS_NONE 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) #define CMDQ_SYNC_0_CS_IRQ 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #define CMDQ_SYNC_0_CS_SEV 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /* Event queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #define EVTQ_ENT_SZ_SHIFT 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) #define EVTQ_0_ID GENMASK_ULL(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /* PRI queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) #define PRIQ_ENT_SZ_SHIFT 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) #define PRIQ_0_SID GENMASK_ULL(31, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) #define PRIQ_0_SSID GENMASK_ULL(51, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #define PRIQ_0_PERM_PRIV (1UL << 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) #define PRIQ_0_PERM_EXEC (1UL << 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) #define PRIQ_0_PERM_READ (1UL << 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #define PRIQ_0_PERM_WRITE (1UL << 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #define PRIQ_0_PRG_LAST (1UL << 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #define PRIQ_0_SSID_V (1UL << 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* High-level queue structures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) #define ARM_SMMU_POLL_SPIN_COUNT 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) #define MSI_IOVA_BASE 0x8000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) #define MSI_IOVA_LENGTH 0x100000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) enum pri_resp {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) PRI_RESP_DENY = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) PRI_RESP_FAIL = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) PRI_RESP_SUCC = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) struct arm_smmu_cmdq_ent {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* Common fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) u8 opcode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) bool substream_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* Command-specific fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) #define CMDQ_OP_PREFETCH_CFG 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) u32 sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) u8 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) } prefetch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #define CMDQ_OP_CFGI_STE 0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) #define CMDQ_OP_CFGI_ALL 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) #define CMDQ_OP_CFGI_CD 0x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) #define CMDQ_OP_CFGI_CD_ALL 0x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) u32 sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) u32 ssid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) bool leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) u8 span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) } cfgi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #define CMDQ_OP_TLBI_NH_ASID 0x11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) #define CMDQ_OP_TLBI_NH_VA 0x12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) #define CMDQ_OP_TLBI_EL2_ALL 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) #define CMDQ_OP_TLBI_S12_VMALL 0x28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) #define CMDQ_OP_TLBI_S2_IPA 0x2a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) #define CMDQ_OP_TLBI_NSNH_ALL 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) u8 num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) u8 scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) u16 asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) u16 vmid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) bool leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) u8 ttl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) u8 tg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) } tlbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) #define CMDQ_OP_ATC_INV 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) #define ATC_INV_SIZE_ALL 52
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) u32 sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) u32 ssid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) u8 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) bool global;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) } atc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) #define CMDQ_OP_PRI_RESP 0x41
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) u32 sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) u32 ssid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) u16 grpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) enum pri_resp resp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) } pri;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) #define CMDQ_OP_CMD_SYNC 0x46
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) u64 msiaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) } sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) struct arm_smmu_ll_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) u32 prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) u32 cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) atomic_t prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) atomic_t cons;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) } atomic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) u8 __pad[SMP_CACHE_BYTES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) } ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) u32 max_n_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct arm_smmu_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) struct arm_smmu_ll_queue llq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) int irq; /* Wired interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) __le64 *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) dma_addr_t base_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) u64 q_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) size_t ent_dwords;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) u32 __iomem *prod_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) u32 __iomem *cons_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) struct arm_smmu_queue_poll {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) ktime_t timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) unsigned int delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) unsigned int spin_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) bool wfe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct arm_smmu_cmdq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct arm_smmu_queue q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) atomic_long_t *valid_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) atomic_t owner_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) atomic_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct arm_smmu_cmdq_batch {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct arm_smmu_evtq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct arm_smmu_queue q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) u32 max_stalls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) struct arm_smmu_priq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) struct arm_smmu_queue q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* High-level stream table and context descriptor structures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct arm_smmu_strtab_l1_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) u8 span;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) __le64 *l2ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) dma_addr_t l2ptr_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct arm_smmu_ctx_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) u16 asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) u64 ttbr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) u64 tcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) u64 mair;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) refcount_t refs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) struct arm_smmu_l1_ctx_desc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) __le64 *l2ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) dma_addr_t l2ptr_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct arm_smmu_ctx_desc_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) __le64 *cdtab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) dma_addr_t cdtab_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct arm_smmu_l1_ctx_desc *l1_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) unsigned int num_l1_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct arm_smmu_s1_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) struct arm_smmu_ctx_desc_cfg cdcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct arm_smmu_ctx_desc cd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) u8 s1fmt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) u8 s1cdmax;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct arm_smmu_s2_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) u16 vmid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) u64 vttbr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) u64 vtcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct arm_smmu_strtab_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) __le64 *strtab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) dma_addr_t strtab_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct arm_smmu_strtab_l1_desc *l1_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) unsigned int num_l1_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) u64 strtab_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) u32 strtab_base_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) /* An SMMUv3 instance */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) struct arm_smmu_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) void __iomem *page1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) #define ARM_SMMU_FEAT_TT_LE (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) #define ARM_SMMU_FEAT_TT_BE (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) #define ARM_SMMU_FEAT_PRI (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) #define ARM_SMMU_FEAT_ATS (1 << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) #define ARM_SMMU_FEAT_SEV (1 << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) #define ARM_SMMU_FEAT_MSI (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) #define ARM_SMMU_FEAT_STALLS (1 << 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) #define ARM_SMMU_FEAT_HYP (1 << 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) #define ARM_SMMU_FEAT_VAX (1 << 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) #define ARM_SMMU_FEAT_RANGE_INV (1 << 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) #define ARM_SMMU_FEAT_BTM (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) #define ARM_SMMU_FEAT_SVA (1 << 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) u32 features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) #define ARM_SMMU_OPT_MSIPOLL (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) u32 options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) struct arm_smmu_cmdq cmdq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct arm_smmu_evtq evtq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct arm_smmu_priq priq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) int gerr_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) int combined_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) unsigned long ias; /* IPA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) unsigned long oas; /* PA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) unsigned long pgsize_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) #define ARM_SMMU_MAX_ASIDS (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) unsigned int asid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) #define ARM_SMMU_MAX_VMIDS (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned int vmid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) unsigned int ssid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) unsigned int sid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) struct arm_smmu_strtab_cfg strtab_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) /* IOMMU core code handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) struct iommu_device iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /* SMMU private data for each master */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) struct arm_smmu_master {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct arm_smmu_device *smmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct arm_smmu_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct list_head domain_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) u32 *sids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) unsigned int num_sids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) bool ats_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) bool sva_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) struct list_head bonds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) unsigned int ssid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /* SMMU private data for an IOMMU domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) enum arm_smmu_domain_stage {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) ARM_SMMU_DOMAIN_S1 = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) ARM_SMMU_DOMAIN_S2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) ARM_SMMU_DOMAIN_NESTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) ARM_SMMU_DOMAIN_BYPASS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct arm_smmu_domain {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct arm_smmu_device *smmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct mutex init_mutex; /* Protects smmu pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) struct io_pgtable_ops *pgtbl_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) bool non_strict;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) atomic_t nr_ats_masters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) enum arm_smmu_domain_stage stage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct arm_smmu_s1_cfg s1_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) struct arm_smmu_s2_cfg s2_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) struct iommu_domain domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) struct list_head devices;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) spinlock_t devices_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) extern struct xarray arm_smmu_asid_xa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) extern struct mutex arm_smmu_asid_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) struct arm_smmu_ctx_desc *cd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) #ifdef CONFIG_ARM_SMMU_V3_SVA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) #else /* CONFIG_ARM_SMMU_V3_SVA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) #endif /* CONFIG_ARM_SMMU_V3_SVA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #endif /* _ARM_SMMU_V3_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags