^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * arch/powerpc/sysdev/dart_iommu.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * IBM Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Based on pSeries_iommu.c:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/suspend.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/kmemleak.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/prom.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/pci-bridge.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/machdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/ppc-pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include "dart.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) /* DART table address and size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static u32 *dart_tablebase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static unsigned long dart_tablesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* Mapped base address for the dart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static unsigned int __iomem *dart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) /* Dummy val that entries are set to when unused */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static unsigned int dart_emptyval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static struct iommu_table iommu_table_dart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static int iommu_table_dart_inited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static int dart_dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static int dart_is_u4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define DART_U4_BYPASS_BASE 0x8000000000ull
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define DBG(...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) static DEFINE_SPINLOCK(invalidate_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static inline void dart_tlb_invalidate_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned long l = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) unsigned int reg, inv_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned long limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) spin_lock_irqsave(&invalidate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) DBG("dart: flush\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * control register and wait for it to clear.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Gotcha: Sometimes, the DART won't detect that the bit gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * set. If so, clear it and set it again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) limit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) l = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) reg = DART_IN(DART_CNTL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) reg |= inv_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) DART_OUT(DART_CNTL, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) l++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (l == (1L << limit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (limit < 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) limit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) reg = DART_IN(DART_CNTL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) reg &= ~inv_bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) DART_OUT(DART_CNTL, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) panic("DART: TLB did not flush after waiting a long "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) "time. Buggy U3 ?");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) spin_unlock_irqrestore(&invalidate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned int reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned int l, limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) spin_lock_irqsave(&invalidate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) (bus_rpn & DART_CNTL_U4_IONE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) DART_OUT(DART_CNTL, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) limit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) wait_more:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) l = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) l++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (l == (1L << limit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (limit < 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) limit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) goto wait_more;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) panic("DART: TLB did not flush after waiting a long "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) "time. Buggy U4 ?");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) spin_unlock_irqrestore(&invalidate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void dart_cache_sync(unsigned int *base, unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * We add 1 to the number of entries to flush, following a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * comment in Darwin indicating that the memory controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * can prefetch unmapped memory under some circumstances.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) unsigned long start = (unsigned long)base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned long end = start + (count + 1) * sizeof(unsigned int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) unsigned int tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /* Perform a standard cache flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) flush_dcache_range(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Perform the sequence described in the CPC925 manual to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * ensure all the data gets to a point the cache incoherent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * DART hardware will see.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) asm volatile(" sync;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) " isync;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) " dcbf 0,%1;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) " sync;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) " isync;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) " lwz %0,0(%1);"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) " isync" : "=r" (tmp) : "r" (end) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static void dart_flush(struct iommu_table *tbl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (dart_dirty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) dart_tlb_invalidate_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) dart_dirty = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static int dart_build(struct iommu_table *tbl, long index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) long npages, unsigned long uaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) enum dma_data_direction direction,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) unsigned int *dp, *orig_dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) unsigned int rpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) long l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) orig_dp = dp = ((unsigned int*)tbl->it_base) + index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* On U3, all memory is contiguous, so we can move this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * out of the loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) l = npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) while (l--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) rpn = __pa(uaddr) >> DART_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) uaddr += DART_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) dart_cache_sync(orig_dp, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (dart_is_u4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) rpn = index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) while (npages--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) dart_tlb_invalidate_one(rpn++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) dart_dirty = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static void dart_free(struct iommu_table *tbl, long index, long npages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) unsigned int *dp, *orig_dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) long orig_npages = npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* We don't worry about flushing the TLB cache. The only drawback of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * not doing it is that we won't catch buggy device drivers doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * bad DMAs, but then no 32-bit architecture ever does either.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) DBG("dart: free at: %lx, %lx\n", index, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) orig_dp = dp = ((unsigned int *)tbl->it_base) + index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) while (npages--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) *(dp++) = dart_emptyval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) dart_cache_sync(orig_dp, orig_npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static void allocate_dart(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) unsigned long tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* 512 pages (2MB) is max DART tablesize. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) dart_tablesize = 1UL << 21;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * will blow up an entire large page anyway in the kernel mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) dart_tablebase = memblock_alloc_try_nid_raw(SZ_16M, SZ_16M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) MEMBLOCK_LOW_LIMIT, SZ_2G,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (!dart_tablebase)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) panic("Failed to allocate 16MB below 2GB for DART table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /* There is no point scanning the DART space for leaks*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) kmemleak_no_scan((void *)dart_tablebase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /* Allocate a spare page to map all invalid DART pages. We need to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * that to work around what looks like a problem with the HT bridge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * prefetching into invalid pages and corrupting data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) tmp = memblock_phys_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (!tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) panic("DART: table allocation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) DARTMAP_RPNMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) printk(KERN_INFO "DART table allocated at: %p\n", dart_tablebase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static int __init dart_init(struct device_node *dart_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) unsigned long base, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct resource r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* IOMMU disabled by the user ? bail out */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (iommu_is_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * Only use the DART if the machine has more than 1GB of RAM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * or if requested with iommu=on on cmdline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * 1GB of RAM is picked as limit because some default devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * (i.e. Airport Extreme) have 30 bit address range limits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* Get DART registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (of_address_to_resource(dart_node, 0, &r))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) panic("DART: can't get register base ! ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /* Map in DART registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) dart = ioremap(r.start, resource_size(&r));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (dart == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) panic("DART: Cannot map registers!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* Allocate the DART and dummy page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) allocate_dart();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* Fill initial table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) for (i = 0; i < dart_tablesize/4; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) dart_tablebase[i] = dart_emptyval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Push to memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* Initialize DART with table base and enable it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) base = ((unsigned long)dart_tablebase) >> DART_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) size = dart_tablesize >> DART_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (dart_is_u4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) size &= DART_SIZE_U4_SIZE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) DART_OUT(DART_BASE_U4, base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) DART_OUT(DART_SIZE_U4, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) size &= DART_CNTL_U3_SIZE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) DART_OUT(DART_CNTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) DART_CNTL_U3_ENABLE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) (base << DART_CNTL_U3_BASE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) (size << DART_CNTL_U3_SIZE_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /* Invalidate DART to get rid of possible stale TLBs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) dart_tlb_invalidate_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) dart_is_u4 ? "U4" : "U3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static struct iommu_table_ops iommu_dart_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) .set = dart_build,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) .clear = dart_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) .flush = dart_flush,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static void iommu_table_dart_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) iommu_table_dart.it_busno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) iommu_table_dart.it_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /* it_size is in number of entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) /* Initialize the common IOMMU code */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) iommu_table_dart.it_base = (unsigned long)dart_tablebase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) iommu_table_dart.it_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) iommu_table_dart.it_blocksize = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) iommu_table_dart.it_ops = &iommu_dart_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) iommu_init_table(&iommu_table_dart, -1, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /* Reserve the last page of the DART to avoid possible prefetch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * past the DART mapped area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static void pci_dma_bus_setup_dart(struct pci_bus *bus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (!iommu_table_dart_inited) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) iommu_table_dart_inited = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) iommu_table_dart_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static bool dart_device_on_pcie(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) struct device_node *np = of_node_get(dev->of_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) while(np) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (of_device_is_compatible(np, "U4-pcie") ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) of_device_is_compatible(np, "u4-pcie")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) of_node_put(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) np = of_get_next_parent(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) static void pci_dma_dev_setup_dart(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (dart_is_u4 && dart_device_on_pcie(&dev->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) dev->dev.archdata.dma_offset = DART_U4_BYPASS_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) set_iommu_table_base(&dev->dev, &iommu_table_dart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) static bool iommu_bypass_supported_dart(struct pci_dev *dev, u64 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return dart_is_u4 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) dart_device_on_pcie(&dev->dev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) mask >= DMA_BIT_MASK(40);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct device_node *dn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) /* Find the DART in the device-tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) dn = of_find_compatible_node(NULL, "dart", "u3-dart");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (dn == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) dn = of_find_compatible_node(NULL, "dart", "u4-dart");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (dn == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return; /* use default direct_dma_ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) dart_is_u4 = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* Initialize the DART HW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (dart_init(dn) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * U4 supports a DART bypass, we use it for 64-bit capable devices to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * improve performance. However, that only works for devices connected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * to the U4 own PCIe interface, not bridged through hypertransport.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * We need the device to support at least 40 bits of addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) controller_ops->dma_dev_setup = pci_dma_dev_setup_dart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) controller_ops->dma_bus_setup = pci_dma_bus_setup_dart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) controller_ops->iommu_bypass_supported = iommu_bypass_supported_dart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* Setup pci_dma ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) set_pci_dma_ops(&dma_iommu_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) #ifdef CONFIG_PM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static void iommu_dart_restore(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) dart_cache_sync(dart_tablebase, dart_tablesize / sizeof(u32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) dart_tlb_invalidate_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static int __init iommu_init_late_dart(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (!dart_tablebase)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) ppc_md.iommu_restore = iommu_dart_restore;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) late_initcall(iommu_init_late_dart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) #endif /* CONFIG_PM */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags