^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * IOMMU API for ARM architected SMMU implementations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013 ARM Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Will Deacon <will.deacon@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #ifndef _ARM_SMMU_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define _ARM_SMMU_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/bitfield.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/bits.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/io-64-nonatomic-hi-lo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/io-pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/irqreturn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* Configuration registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define ARM_SMMU_GR0_sCR0 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define ARM_SMMU_sCR0_VMID16EN BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define ARM_SMMU_sCR0_BSU GENMASK(15, 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define ARM_SMMU_sCR0_FB BIT(13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define ARM_SMMU_sCR0_PTM BIT(12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define ARM_SMMU_sCR0_VMIDPNE BIT(11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define ARM_SMMU_sCR0_USFCFG BIT(10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define ARM_SMMU_sCR0_GCFGFIE BIT(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define ARM_SMMU_sCR0_GCFGFRE BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define ARM_SMMU_sCR0_EXIDENABLE BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define ARM_SMMU_sCR0_GFIE BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define ARM_SMMU_sCR0_GFRE BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define ARM_SMMU_sCR0_CLIENTPD BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* Auxiliary Configuration register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define ARM_SMMU_GR0_sACR 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /* Identification registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define ARM_SMMU_GR0_ID0 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define ARM_SMMU_ID0_S1TS BIT(30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define ARM_SMMU_ID0_S2TS BIT(29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define ARM_SMMU_ID0_NTS BIT(28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define ARM_SMMU_ID0_SMS BIT(27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define ARM_SMMU_ID0_ATOSNS BIT(26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define ARM_SMMU_ID0_CTTW BIT(14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define ARM_SMMU_ID0_EXIDS BIT(8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define ARM_SMMU_GR0_ID1 0x24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define ARM_SMMU_ID1_PAGESIZE BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define ARM_SMMU_ID1_NUMCB GENMASK(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define ARM_SMMU_GR0_ID2 0x28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define ARM_SMMU_ID2_VMID16 BIT(15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define ARM_SMMU_ID2_PTFS_64K BIT(14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define ARM_SMMU_ID2_PTFS_16K BIT(13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define ARM_SMMU_ID2_PTFS_4K BIT(12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define ARM_SMMU_ID2_UBS GENMASK(11, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define ARM_SMMU_ID2_OAS GENMASK(7, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define ARM_SMMU_ID2_IAS GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define ARM_SMMU_GR0_ID3 0x2c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define ARM_SMMU_GR0_ID4 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define ARM_SMMU_GR0_ID5 0x34
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define ARM_SMMU_GR0_ID6 0x38
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define ARM_SMMU_GR0_ID7 0x3c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define ARM_SMMU_ID7_MAJOR GENMASK(7, 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define ARM_SMMU_ID7_MINOR GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define ARM_SMMU_GR0_sGFSR 0x48
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define ARM_SMMU_sGFSR_USF BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define ARM_SMMU_GR0_sGFSYNR0 0x50
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define ARM_SMMU_GR0_sGFSYNR1 0x54
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define ARM_SMMU_GR0_sGFSYNR2 0x58
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* Global TLB invalidation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define ARM_SMMU_GR0_TLBIVMID 0x64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define ARM_SMMU_GR0_TLBIALLH 0x6c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define ARM_SMMU_GR0_sTLBGSYNC 0x70
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* Stream mapping registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define ARM_SMMU_SMR_VALID BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define ARM_SMMU_SMR_MASK GENMASK(31, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define ARM_SMMU_SMR_ID GENMASK(15, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) enum arm_smmu_s2cr_privcfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) S2CR_PRIVCFG_DEFAULT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) S2CR_PRIVCFG_DIPAN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) S2CR_PRIVCFG_UNPRIV,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) S2CR_PRIVCFG_PRIV,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define ARM_SMMU_S2CR_TYPE GENMASK(17, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) enum arm_smmu_s2cr_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) S2CR_TYPE_TRANS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) S2CR_TYPE_BYPASS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) S2CR_TYPE_FAULT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define ARM_SMMU_S2CR_EXIDVALID BIT(10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /* Context bank attribute registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define ARM_SMMU_CBAR_TYPE GENMASK(17, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) enum arm_smmu_cbar_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) CBAR_TYPE_S2_TRANS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) CBAR_TYPE_S1_TRANS_S2_BYPASS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) CBAR_TYPE_S1_TRANS_S2_FAULT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) CBAR_TYPE_S1_TRANS_S2_TRANS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define ARM_SMMU_CBAR_VMID GENMASK(7, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define ARM_SMMU_CBA2R_VA64 BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #define ARM_SMMU_CB_SCTLR 0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #define ARM_SMMU_SCTLR_CFCFG BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #define ARM_SMMU_SCTLR_CFIE BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define ARM_SMMU_SCTLR_CFRE BIT(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define ARM_SMMU_SCTLR_E BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define ARM_SMMU_SCTLR_AFE BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define ARM_SMMU_SCTLR_TRE BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define ARM_SMMU_SCTLR_M BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define ARM_SMMU_CB_ACTLR 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define ARM_SMMU_CB_RESUME 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #define ARM_SMMU_RESUME_TERMINATE BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define ARM_SMMU_CB_TCR2 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define ARM_SMMU_TCR2_SEP GENMASK(17, 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) #define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define ARM_SMMU_TCR2_AS BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define ARM_SMMU_CB_TTBR0 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #define ARM_SMMU_CB_TTBR1 0x28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define ARM_SMMU_CB_TCR 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #define ARM_SMMU_TCR_EAE BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #define ARM_SMMU_TCR_EPD1 BIT(23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #define ARM_SMMU_TCR_A1 BIT(22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #define ARM_SMMU_TCR_TG0 GENMASK(15, 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #define ARM_SMMU_TCR_SH0 GENMASK(13, 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define ARM_SMMU_TCR_EPD0 BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define ARM_SMMU_TCR_T0SZ GENMASK(5, 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #define ARM_SMMU_VTCR_RES1 BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #define ARM_SMMU_VTCR_PS GENMASK(18, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #define ARM_SMMU_VTCR_SL0 GENMASK(7, 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) #define ARM_SMMU_CB_CONTEXTIDR 0x34
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #define ARM_SMMU_CB_S1_MAIR0 0x38
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) #define ARM_SMMU_CB_S1_MAIR1 0x3c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define ARM_SMMU_CB_PAR 0x50
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define ARM_SMMU_CB_PAR_F BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #define ARM_SMMU_CB_FSR 0x58
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #define ARM_SMMU_FSR_MULTI BIT(31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define ARM_SMMU_FSR_SS BIT(30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #define ARM_SMMU_FSR_UUT BIT(8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #define ARM_SMMU_FSR_ASF BIT(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #define ARM_SMMU_FSR_TLBLKF BIT(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define ARM_SMMU_FSR_TLBMCF BIT(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define ARM_SMMU_FSR_EF BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #define ARM_SMMU_FSR_PF BIT(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #define ARM_SMMU_FSR_AFF BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #define ARM_SMMU_FSR_TF BIT(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) ARM_SMMU_FSR_ASF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) ARM_SMMU_FSR_TLBMCF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) ARM_SMMU_FSR_TLBLKF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) ARM_SMMU_FSR_SS | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ARM_SMMU_FSR_UUT | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) ARM_SMMU_FSR_EF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) ARM_SMMU_FSR_PF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) ARM_SMMU_FSR_TF | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) ARM_SMMU_FSR_IGN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) #define ARM_SMMU_CB_FAR 0x60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #define ARM_SMMU_CB_FSYNR0 0x68
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #define ARM_SMMU_FSYNR0_WNR BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #define ARM_SMMU_CB_S1_TLBIVA 0x600
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #define ARM_SMMU_CB_S1_TLBIASID 0x610
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #define ARM_SMMU_CB_S1_TLBIVAL 0x620
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) #define ARM_SMMU_CB_TLBSYNC 0x7f0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #define ARM_SMMU_CB_TLBSTATUS 0x7f4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #define ARM_SMMU_CB_ATS1PR 0x800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) #define ARM_SMMU_CB_ATSR 0x8f0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #define ARM_SMMU_ATSR_ACTIVE BIT(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* Maximum number of context banks per SMMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) #define ARM_SMMU_MAX_CBS 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define TLB_SPIN_COUNT 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /* Shared driver definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) enum arm_smmu_arch_version {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) ARM_SMMU_V1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) ARM_SMMU_V1_64K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ARM_SMMU_V2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) enum arm_smmu_implementation {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) GENERIC_SMMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) ARM_MMU500,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) CAVIUM_SMMUV2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) QCOM_SMMUV2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct arm_smmu_s2cr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct iommu_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) enum arm_smmu_s2cr_type type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) enum arm_smmu_s2cr_privcfg privcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) u8 cbndx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) bool pinned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct arm_smmu_smr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) u16 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) u16 id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) bool valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct arm_smmu_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) void __iomem *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) unsigned int numpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) unsigned int pgshift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) #define ARM_SMMU_FEAT_VMID16 (1 << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #define ARM_SMMU_FEAT_EXIDS (1 << 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) u32 features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) enum arm_smmu_arch_version version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) enum arm_smmu_implementation model;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) const struct arm_smmu_impl *impl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) u32 num_context_banks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) u32 num_s2_context_banks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct arm_smmu_cb *cbs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) atomic_t irptndx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) u32 num_mapping_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) u16 streamid_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) u16 smr_mask_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct arm_smmu_smr *smrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct arm_smmu_s2cr *s2crs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct mutex stream_map_mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) unsigned long va_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) unsigned long ipa_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) unsigned long pa_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) unsigned long pgsize_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) u32 num_global_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) u32 num_context_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) unsigned int *irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct clk_bulk_data *clks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) int num_clks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) spinlock_t global_sync_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* IOMMU core code handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) struct iommu_device iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) enum arm_smmu_context_fmt {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) ARM_SMMU_CTX_FMT_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ARM_SMMU_CTX_FMT_AARCH64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) ARM_SMMU_CTX_FMT_AARCH32_L,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ARM_SMMU_CTX_FMT_AARCH32_S,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct arm_smmu_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) u8 cbndx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) u8 irptndx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) u16 asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) u16 vmid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) enum arm_smmu_cbar_type cbar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) enum arm_smmu_context_fmt fmt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #define ARM_SMMU_INVALID_IRPTNDX 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct arm_smmu_cb {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) u64 ttbr[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) u32 tcr[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) u32 mair[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct arm_smmu_cfg *cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) enum arm_smmu_domain_stage {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) ARM_SMMU_DOMAIN_S1 = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) ARM_SMMU_DOMAIN_S2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) ARM_SMMU_DOMAIN_NESTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ARM_SMMU_DOMAIN_BYPASS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) struct arm_smmu_domain {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct arm_smmu_device *smmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) struct io_pgtable_ops *pgtbl_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) const struct iommu_flush_ops *flush_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct arm_smmu_cfg cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) enum arm_smmu_domain_stage stage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) bool non_strict;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct mutex init_mutex; /* Protects smmu pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct iommu_domain domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct arm_smmu_master_cfg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct arm_smmu_device *smmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) s16 smendx[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * When TTBR1 is selected shift the TCR fields by 16 bits and disable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * translation in TTBR0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) tcr |= ARM_SMMU_TCR_EPD0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) tcr |= ARM_SMMU_TCR_EPD1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return tcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return ARM_SMMU_VTCR_RES1 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* Implementation details, yay! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct arm_smmu_impl {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) u32 val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) u64 val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int (*cfg_probe)(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) int (*reset)(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) int (*init_context)(struct arm_smmu_domain *smmu_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct io_pgtable_cfg *cfg, struct device *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) int status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) int (*def_domain_type)(struct device *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) irqreturn_t (*global_fault)(int irq, void *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) irqreturn_t (*context_fault)(int irq, void *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct arm_smmu_device *smmu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct device *dev, int start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) void (*write_s2cr)(struct arm_smmu_device *smmu, int idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #define INVALID_SMENDX -1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) #define cfg_smendx(cfg, fw, i) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) (i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) #define for_each_cfg_sme(cfg, fw, i, idx) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) idx = find_next_zero_bit(map, end, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (idx == end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) } while (test_and_set_bit(idx, map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return smmu->base + (n << smmu->pgshift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (smmu->impl && unlikely(smmu->impl->read_reg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) return smmu->impl->read_reg(smmu, page, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return readl_relaxed(arm_smmu_page(smmu, page) + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) int offset, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (smmu->impl && unlikely(smmu->impl->write_reg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) smmu->impl->write_reg(smmu, page, offset, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (smmu->impl && unlikely(smmu->impl->read_reg64))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return smmu->impl->read_reg64(smmu, page, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return readq_relaxed(arm_smmu_page(smmu, page) + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) int offset, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (smmu->impl && unlikely(smmu->impl->write_reg64))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) smmu->impl->write_reg64(smmu, page, offset, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) #define ARM_SMMU_GR0 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) #define ARM_SMMU_GR1 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) #define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) #define arm_smmu_gr0_read(s, o) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) arm_smmu_readl((s), ARM_SMMU_GR0, (o))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) #define arm_smmu_gr0_write(s, o, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) #define arm_smmu_gr1_read(s, o) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) arm_smmu_readl((s), ARM_SMMU_GR1, (o))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) #define arm_smmu_gr1_write(s, o, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) #define arm_smmu_cb_read(s, n, o) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) #define arm_smmu_cb_write(s, n, o, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) #define arm_smmu_cb_readq(s, n, o) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) #define arm_smmu_cb_writeq(s, n, o, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) struct arm_smmu_device *qcom_adreno_smmu_impl_init(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) int arm_mmu500_reset(struct arm_smmu_device *smmu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) #endif /* _ARM_SMMU_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags