^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * CPU-agnostic ARM page table allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2014 ARM Limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Will Deacon <will.deacon@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/io-pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include "io-pgtable-arm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define ARM_LPAE_MAX_ADDR_BITS 52
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define ARM_LPAE_MAX_LEVELS 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* Struct accessors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define io_pgtable_to_data(x) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) container_of((x), struct arm_lpae_io_pgtable, iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define io_pgtable_ops_to_data(x) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * Calculate the right shift amount to get to the portion describing level l
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * in a virtual address mapped by the pagetable in d.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define ARM_LPAE_LVL_SHIFT(l,d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) ilog2(sizeof(arm_lpae_iopte)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define ARM_LPAE_GRANULE(d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define ARM_LPAE_PGD_SIZE(d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define ARM_LPAE_PTES_PER_TABLE(d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * Calculate the index at level l used to map virtual address a using the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * pagetable in d.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define ARM_LPAE_PGD_IDX(l,d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define ARM_LPAE_LVL_IDX(a,l,d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* Calculate the block/page mapping size at level l for pagetable in d. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Page table bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define ARM_LPAE_PTE_TYPE_SHIFT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define ARM_LPAE_PTE_TYPE_MASK 0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define ARM_LPAE_PTE_TYPE_BLOCK 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define ARM_LPAE_PTE_TYPE_TABLE 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define ARM_LPAE_PTE_TYPE_PAGE 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* Ignore the contiguous bit for block splitting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) ARM_LPAE_PTE_ATTR_HI_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* Software bit for solving coherency races */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /* Stage-1 PTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* Stage-2 PTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Register bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #define ARM_LPAE_VTCR_SL0_MASK 0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define ARM_LPAE_TCR_T0SZ_SHIFT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define ARM_LPAE_VTCR_PS_SHIFT 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define ARM_LPAE_VTCR_PS_MASK 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define ARM_LPAE_MAIR_ATTR_MASK 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #define ARM_LPAE_MAIR_ATTR_NC 0x44ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define ARM_LPAE_MAIR_ATTR_INC_OWBRANWA 0xe4ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA 0xefULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define ARM_LPAE_MAIR_ATTR_WBRWA 0xffULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #define ARM_LPAE_MAIR_ATTR_IDX_NC 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* IOPTE accessors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define iopte_type(pte,l) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct arm_lpae_io_pgtable {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct io_pgtable iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) int pgd_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) int start_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) int bits_per_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) void *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) typedef u64 arm_lpae_iopte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) enum io_pgtable_fmt fmt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct arm_lpae_io_pgtable *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) arm_lpae_iopte pte = paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct arm_lpae_io_pgtable *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (ARM_LPAE_GRANULE(data) < SZ_64K)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /* Rotate the packed high-order bits back to the top */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static bool selftest_running = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static dma_addr_t __arm_lpae_dma_addr(void *pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return (dma_addr_t)virt_to_phys(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct device *dev = cfg->iommu_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) dma_addr_t dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) void *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) VM_BUG_ON((gfp & __GFP_HIGHMEM));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) gfp | __GFP_ZERO, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) pages = page_address(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (!cfg->coherent_walk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (dma_mapping_error(dev, dma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * We depend on the IOMMU being able to work with any physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * address directly, so if the DMA layer suggests otherwise by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * translating or truncating them, that bodes very badly...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (dma != virt_to_phys(pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) goto out_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) out_unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) __free_pages(p, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static void __arm_lpae_free_pages(void *pages, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (!cfg->coherent_walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) size, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) free_pages((unsigned long)pages, get_order(size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) *ptep = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (!cfg->coherent_walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) __arm_lpae_sync_pte(ptep, 1, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct iommu_iotlb_gather *gather,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) unsigned long iova, size_t size, size_t pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) int lvl, arm_lpae_iopte *ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) phys_addr_t paddr, arm_lpae_iopte prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int lvl, int num_entries, arm_lpae_iopte *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) arm_lpae_iopte pte = prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pte |= ARM_LPAE_PTE_TYPE_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) pte |= ARM_LPAE_PTE_TYPE_BLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) for (i = 0; i < num_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) if (!cfg->coherent_walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) __arm_lpae_sync_pte(ptep, num_entries, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) unsigned long iova, phys_addr_t paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) arm_lpae_iopte prot, int lvl, int num_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) arm_lpae_iopte *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) for (i = 0; i < num_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* We require an unmap first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) WARN_ON(!selftest_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) } else if (iopte_type(ptep[i], lvl) == ARM_LPAE_PTE_TYPE_TABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * We need to unmap and free the old table before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * overwriting it with a block entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) arm_lpae_iopte *tblp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) lvl, tblp) != sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) arm_lpae_iopte *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) arm_lpae_iopte curr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct arm_lpae_io_pgtable *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) arm_lpae_iopte old, new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) new |= ARM_LPAE_PTE_NSTABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * Ensure the table itself is visible before its PTE can be.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * Whilst we could get away with cmpxchg64_release below, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * doesn't have any ordering semantics when !CONFIG_SMP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) dma_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) old = cmpxchg64_relaxed(ptep, curr, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (cfg->coherent_walk || (old & ARM_LPAE_PTE_SW_SYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* Even if it's not ours, there's no point waiting; just kick it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) __arm_lpae_sync_pte(ptep, 1, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (old == curr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) WRITE_ONCE(*ptep, new | ARM_LPAE_PTE_SW_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) phys_addr_t paddr, size_t size, size_t pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) gfp_t gfp, size_t *mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) arm_lpae_iopte *cptep, pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) size_t tblsz = ARM_LPAE_GRANULE(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) int ret = 0, num_entries, max_entries, map_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) /* Find our entry at the current level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) ptep += map_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) /* If we can install a leaf entry at this level, then do so */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (size == block_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) num_entries = min_t(int, pgcount, max_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (!ret && mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) *mapped += num_entries * size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /* We can't allocate tables at the final level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* Grab a pointer to the next level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (!pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!cptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) pte = arm_lpae_install_table(cptep, ptep, 0, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) __arm_lpae_free_pages(cptep, tblsz, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) __arm_lpae_sync_pte(ptep, 1, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) cptep = iopte_deref(pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) } else if (pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /* We require an unmap first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) WARN_ON(!selftest_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /* Rinse, repeat */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) cptep, gfp, mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) arm_lpae_iopte pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (data->iop.fmt == ARM_64_LPAE_S1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) data->iop.fmt == ARM_32_LPAE_S1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) pte = ARM_LPAE_PTE_nG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) pte |= ARM_LPAE_PTE_AP_RDONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!(prot & IOMMU_PRIV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) pte |= ARM_LPAE_PTE_AP_UNPRIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) pte = ARM_LPAE_PTE_HAP_FAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (prot & IOMMU_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) pte |= ARM_LPAE_PTE_HAP_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (prot & IOMMU_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) pte |= ARM_LPAE_PTE_HAP_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * Note that this logic is structured to accommodate Mali LPAE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * having stage-1-like attributes but stage-2-like permissions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (data->iop.fmt == ARM_64_LPAE_S2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) data->iop.fmt == ARM_32_LPAE_S2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (prot & IOMMU_MMIO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) pte |= ARM_LPAE_PTE_MEMATTR_DEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) else if (prot & IOMMU_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) pte |= ARM_LPAE_PTE_MEMATTR_NC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (prot & IOMMU_MMIO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) << ARM_LPAE_PTE_ATTRINDX_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) else if ((prot & IOMMU_CACHE) && (prot & IOMMU_SYS_CACHE_NWA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) pte |= (ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) << ARM_LPAE_PTE_ATTRINDX_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* IOMMU_CACHE + IOMMU_SYS_CACHE equivalent to IOMMU_CACHE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) else if (prot & IOMMU_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) << ARM_LPAE_PTE_ATTRINDX_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) else if (prot & IOMMU_SYS_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) << ARM_LPAE_PTE_ATTRINDX_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) else if (prot & IOMMU_SYS_CACHE_NWA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) << ARM_LPAE_PTE_ATTRINDX_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * Also Mali has its own notions of shareability wherein its Inner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * domain covers the cores within the GPU, and its Outer domain is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * "outside the GPU" (i.e. either the Inner or System domain in CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * terms, depending on coherency).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) pte |= ARM_LPAE_PTE_SH_IS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) pte |= ARM_LPAE_PTE_SH_OS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (prot & IOMMU_NOEXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) pte |= ARM_LPAE_PTE_XN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) pte |= ARM_LPAE_PTE_NS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (data->iop.fmt != ARM_MALI_LPAE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) pte |= ARM_LPAE_PTE_AF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) phys_addr_t paddr, size_t pgsize, size_t pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) int iommu_prot, gfp_t gfp, size_t *mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) arm_lpae_iopte *ptep = data->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) int ret, lvl = data->start_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) arm_lpae_iopte prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) long iaext = (s64)iova >> cfg->ias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) /* If no access, then nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) iaext = ~iaext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (WARN_ON(iaext || paddr >> cfg->oas))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) prot = arm_lpae_prot_to_pte(data, iommu_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) ptep, gfp, mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * Synchronise all PTE updates for the new mapping before there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * a chance for anything to kick off a table walk for the new iova.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) arm_lpae_iopte *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) arm_lpae_iopte *start, *end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) unsigned long table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (lvl == data->start_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) table_size = ARM_LPAE_PGD_SIZE(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) table_size = ARM_LPAE_GRANULE(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) start = ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* Only leaf entries at the last level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (lvl == ARM_LPAE_MAX_LEVELS - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) end = ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) end = (void *)ptep + table_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) while (ptep != end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) arm_lpae_iopte pte = *ptep++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static void arm_lpae_free_pgtable(struct io_pgtable *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct iommu_iotlb_gather *gather,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unsigned long iova, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) arm_lpae_iopte blk_pte, int lvl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) arm_lpae_iopte *ptep, size_t pgcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) arm_lpae_iopte pte, *tablep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) phys_addr_t blk_paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) size_t tablesz = ARM_LPAE_GRANULE(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) int i, unmap_idx_start = -1, num_entries = 0, max_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (!tablep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return 0; /* Bytes unmapped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (size == split_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) max_entries = ptes_per_table - unmap_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) num_entries = min_t(int, pgcount, max_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) blk_paddr = iopte_to_paddr(blk_pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) pte = iopte_prot(blk_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) for (i = 0; i < ptes_per_table; i++, blk_paddr += split_sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) /* Unmap! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (i >= unmap_idx_start && i < (unmap_idx_start + num_entries))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (pte != blk_pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) __arm_lpae_free_pages(tablep, tablesz, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * We may race against someone unmapping another part of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * block, but anything else is invalid. We can't misinterpret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * a page entry here since we're never at the last level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (iopte_type(pte, lvl - 1) != ARM_LPAE_PTE_TYPE_TABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) tablep = iopte_deref(pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) } else if (unmap_idx_start >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) for (i = 0; i < num_entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return num_entries * size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct iommu_iotlb_gather *gather,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) unsigned long iova, size_t size, size_t pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) int lvl, arm_lpae_iopte *ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) arm_lpae_iopte pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct io_pgtable *iop = &data->iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) int i = 0, num_entries, max_entries, unmap_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) /* Something went horribly wrong and we ran out of page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) ptep += unmap_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (WARN_ON(!pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /* If the size matches this level, we're in the right place */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) num_entries = min_t(int, pgcount, max_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) while (i < num_entries) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (WARN_ON(!pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) __arm_lpae_clear_pte(ptep, &iop->cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (!iopte_leaf(pte, lvl, iop->fmt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /* Also flush any partial walks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) ARM_LPAE_GRANULE(data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * Order the PTE update against queueing the IOVA, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * guarantee that a flush callback from a different CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * has observed it before the TLBIALL can be issued.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) io_pgtable_tlb_add_page(iop, gather, iova + i * size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) ptep++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return i * size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) } else if (iopte_leaf(pte, lvl, iop->fmt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * Insert a table at the next level to map the old region,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * minus the part we want to unmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) lvl + 1, ptep, pgcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /* Keep on walkin' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) ptep = iopte_deref(pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) size_t pgsize, size_t pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct iommu_iotlb_gather *gather)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) arm_lpae_iopte *ptep = data->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) long iaext = (s64)iova >> cfg->ias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) iaext = ~iaext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) if (WARN_ON(iaext))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) data->start_level, ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) size_t size, struct iommu_iotlb_gather *gather)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return arm_lpae_unmap_pages(ops, iova, size, 1, gather);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) unsigned long iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) arm_lpae_iopte pte, *ptep = data->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) int lvl = data->start_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) /* Valid IOPTE pointer? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (!ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /* Grab the IOPTE we're interested in */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /* Valid entry? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* Leaf entry? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (iopte_leaf(pte, lvl, data->iop.fmt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) goto found_translation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /* Take it to the next level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) ptep = iopte_deref(pte, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) } while (++lvl < ARM_LPAE_MAX_LEVELS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /* Ran out of page tables to walk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) found_translation:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) return iopte_to_paddr(pte, data) | iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) unsigned long granule, page_sizes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) unsigned int max_addr_bits = 48;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * We need to restrict the supported page sizes to match the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * translation regime for a particular granule. Aim to match
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * the CPU page size if possible, otherwise prefer smaller sizes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * While we're at it, restrict the block sizes to match the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * chosen granule.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (cfg->pgsize_bitmap & PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) granule = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) else if (cfg->pgsize_bitmap & ~PAGE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) else if (cfg->pgsize_bitmap & PAGE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) granule = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) switch (granule) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) case SZ_4K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) page_sizes = (SZ_4K | SZ_2M | SZ_1G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) case SZ_16K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) page_sizes = (SZ_16K | SZ_32M);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) case SZ_64K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) max_addr_bits = 52;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) page_sizes = (SZ_64K | SZ_512M);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (cfg->oas > 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) page_sizes |= 1ULL << 42; /* 4TB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) page_sizes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) cfg->pgsize_bitmap &= page_sizes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) cfg->ias = min(cfg->ias, max_addr_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) cfg->oas = min(cfg->oas, max_addr_bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) static struct arm_lpae_io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) struct arm_lpae_io_pgtable *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) int levels, va_bits, pg_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) arm_lpae_restrict_pgsizes(cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) data = kmalloc(sizeof(*data), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) pg_shift = __ffs(cfg->pgsize_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) va_bits = cfg->ias - pg_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) data->start_level = ARM_LPAE_MAX_LEVELS - levels;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /* Calculate the actual size of our pgd (without concatenation) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) data->iop.ops = (struct io_pgtable_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) .map = arm_lpae_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) .map_pages = arm_lpae_map_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) .unmap = arm_lpae_unmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) .unmap_pages = arm_lpae_unmap_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) .iova_to_phys = arm_lpae_iova_to_phys,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) return data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) static struct io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) u64 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) struct arm_lpae_io_pgtable *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) bool tg1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) IO_PGTABLE_QUIRK_NON_STRICT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) IO_PGTABLE_QUIRK_ARM_TTBR1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) data = arm_lpae_alloc_pgtable(cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) /* TCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) if (cfg->coherent_walk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) tcr->sh = ARM_LPAE_TCR_SH_IS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) tcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) tcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) tcr->sh = ARM_LPAE_TCR_SH_OS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) tcr->irgn = ARM_LPAE_TCR_RGN_NC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) tcr->orgn = ARM_LPAE_TCR_RGN_NC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) switch (ARM_LPAE_GRANULE(data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) case SZ_4K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_4K : ARM_LPAE_TCR_TG0_4K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) case SZ_16K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_16K : ARM_LPAE_TCR_TG0_16K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) case SZ_64K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) tcr->tg = tg1 ? ARM_LPAE_TCR_TG1_64K : ARM_LPAE_TCR_TG0_64K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) switch (cfg->oas) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) case 32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) tcr->ips = ARM_LPAE_TCR_PS_32_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) case 36:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) tcr->ips = ARM_LPAE_TCR_PS_36_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) case 40:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) tcr->ips = ARM_LPAE_TCR_PS_40_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) case 42:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) tcr->ips = ARM_LPAE_TCR_PS_42_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) case 44:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) tcr->ips = ARM_LPAE_TCR_PS_44_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) case 48:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) tcr->ips = ARM_LPAE_TCR_PS_48_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) case 52:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) tcr->ips = ARM_LPAE_TCR_PS_52_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) goto out_free_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) tcr->tsz = 64ULL - cfg->ias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) /* MAIRs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) reg = (ARM_LPAE_MAIR_ATTR_NC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) (ARM_LPAE_MAIR_ATTR_WBRWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) (ARM_LPAE_MAIR_ATTR_DEVICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) (ARM_LPAE_MAIR_ATTR_INC_OWBRWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) (ARM_LPAE_MAIR_ATTR_INC_OWBRANWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) (ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) cfg->arm_lpae_s1_cfg.mair = reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /* Looking good; allocate a pgd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) GFP_KERNEL, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) if (!data->pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) goto out_free_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) /* Ensure the empty pgd is visible before any actual TTBR write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) /* TTBR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return &data->iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) out_free_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) static struct io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) u64 sl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) struct arm_lpae_io_pgtable *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) /* The NS quirk doesn't apply at stage 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NON_STRICT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) data = arm_lpae_alloc_pgtable(cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * Concatenate PGDs at level 1 if possible in order to reduce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * the depth of the stage-2 walk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (data->start_level == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) unsigned long pgd_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) data->pgd_bits += data->bits_per_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) data->start_level++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* VTCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if (cfg->coherent_walk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) vtcr->sh = ARM_LPAE_TCR_SH_IS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) vtcr->sh = ARM_LPAE_TCR_SH_OS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) vtcr->irgn = ARM_LPAE_TCR_RGN_NC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) vtcr->orgn = ARM_LPAE_TCR_RGN_NC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) sl = data->start_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) switch (ARM_LPAE_GRANULE(data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) case SZ_4K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) vtcr->tg = ARM_LPAE_TCR_TG0_4K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) sl++; /* SL0 format is different for 4K granule size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) case SZ_16K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) vtcr->tg = ARM_LPAE_TCR_TG0_16K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) case SZ_64K:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) vtcr->tg = ARM_LPAE_TCR_TG0_64K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) switch (cfg->oas) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) case 32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) vtcr->ps = ARM_LPAE_TCR_PS_32_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) case 36:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) vtcr->ps = ARM_LPAE_TCR_PS_36_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) case 40:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) vtcr->ps = ARM_LPAE_TCR_PS_40_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) case 42:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) vtcr->ps = ARM_LPAE_TCR_PS_42_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) case 44:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) vtcr->ps = ARM_LPAE_TCR_PS_44_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) case 48:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) vtcr->ps = ARM_LPAE_TCR_PS_48_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) case 52:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) vtcr->ps = ARM_LPAE_TCR_PS_52_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) goto out_free_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) vtcr->tsz = 64ULL - cfg->ias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) vtcr->sl = ~sl & ARM_LPAE_VTCR_SL0_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /* Allocate pgd pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) GFP_KERNEL, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (!data->pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) goto out_free_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /* Ensure the empty pgd is visible before any actual TTBR write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /* VTTBR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return &data->iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) out_free_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) static struct io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (cfg->ias > 32 || cfg->oas > 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) return arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) static struct io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) if (cfg->ias > 40 || cfg->oas > 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return arm_64_lpae_alloc_pgtable_s2(cfg, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) static struct io_pgtable *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct arm_lpae_io_pgtable *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) /* No quirks for Mali (hopefully) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (cfg->quirks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (cfg->ias > 48 || cfg->oas > 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) data = arm_lpae_alloc_pgtable(cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) /* Mali seems to need a full 4-level table regardless of IAS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (data->start_level > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) data->start_level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) data->pgd_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * best we can do is mimic the out-of-tree driver and hope that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * "implementation-defined caching policy" is good enough. Similarly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * we'll use it for the sake of a valid attribute for our 'device'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * index, although callers should never request that in practice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) cfg->arm_mali_lpae_cfg.memattr =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) (ARM_MALI_LPAE_MEMATTR_IMP_DEF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) (ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) (ARM_MALI_LPAE_MEMATTR_IMP_DEF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) if (!data->pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) goto out_free_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) /* Ensure the empty pgd is visible before TRANSTAB can be written */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) ARM_MALI_LPAE_TTBR_READ_INNER |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (cfg->coherent_walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return &data->iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) out_free_data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) .alloc = arm_64_lpae_alloc_pgtable_s1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) .free = arm_lpae_free_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) .alloc = arm_64_lpae_alloc_pgtable_s2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) .free = arm_lpae_free_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) .alloc = arm_32_lpae_alloc_pgtable_s1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) .free = arm_lpae_free_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) .alloc = arm_32_lpae_alloc_pgtable_s2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) .free = arm_lpae_free_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) .alloc = arm_mali_lpae_alloc_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) .free = arm_lpae_free_pgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) static struct io_pgtable_cfg *cfg_cookie __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) static void __init dummy_tlb_flush_all(void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) WARN_ON(cookie != cfg_cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) static void __init dummy_tlb_flush(unsigned long iova, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) size_t granule, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) WARN_ON(cookie != cfg_cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) unsigned long iova, size_t granule,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) dummy_tlb_flush(iova, granule, granule, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) static const struct iommu_flush_ops dummy_tlb_ops __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) .tlb_flush_all = dummy_tlb_flush_all,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) .tlb_flush_walk = dummy_tlb_flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) .tlb_add_page = dummy_tlb_add_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) struct io_pgtable_cfg *cfg = &data->iop.cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) cfg->pgsize_bitmap, cfg->ias);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) #define __FAIL(ops, i) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) arm_lpae_dump_ops(ops); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) selftest_running = false; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) -EFAULT; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) static const enum io_pgtable_fmt fmts[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) ARM_64_LPAE_S1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) ARM_64_LPAE_S2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) unsigned long iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) struct io_pgtable_ops *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) selftest_running = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) cfg_cookie = cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) if (!ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) pr_err("selftest: failed to allocate io pgtable ops\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * Initial sanity checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * Empty page tables shouldn't provide any translations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (ops->iova_to_phys(ops, 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) if (ops->iova_to_phys(ops, SZ_1G + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) if (ops->iova_to_phys(ops, SZ_2G + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * Distinct mappings of different granule sizes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) iova = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) size = 1UL << j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if (ops->map(ops, iova, iova, size, IOMMU_READ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) IOMMU_WRITE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) IOMMU_NOEXEC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) IOMMU_CACHE, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) /* Overlapping mappings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (!ops->map(ops, iova, iova + size, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) iova += SZ_1G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) /* Partial unmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) size = 1UL << __ffs(cfg->pgsize_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (ops->unmap(ops, SZ_1G + size, size, NULL) != size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) /* Remap of partial unmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /* Full unmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) iova = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) size = 1UL << j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (ops->unmap(ops, iova, size, NULL) != size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) if (ops->iova_to_phys(ops, iova + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) /* Remap full block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) return __FAIL(ops, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) iova += SZ_1G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) free_io_pgtable_ops(ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) selftest_running = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) static int __init arm_lpae_do_selftests(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) static const unsigned long pgsize[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) SZ_4K | SZ_2M | SZ_1G,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) SZ_16K | SZ_32M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) SZ_64K | SZ_512M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) static const unsigned int ias[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) 32, 36, 40, 42, 44, 48,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) int i, j, pass = 0, fail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) struct io_pgtable_cfg cfg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) .tlb = &dummy_tlb_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) .oas = 48,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) .coherent_walk = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) for (i = 0; i < ARRAY_SIZE(pgsize); ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) for (j = 0; j < ARRAY_SIZE(ias); ++j) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) cfg.pgsize_bitmap = pgsize[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) cfg.ias = ias[j];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) pgsize[i], ias[j]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) if (arm_lpae_run_tests(&cfg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) fail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) pass++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) return fail ? -EFAULT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) subsys_initcall(arm_lpae_do_selftests);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags