^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright 2010
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * by Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * This code provides a IOMMU for Xen PV guests with PCI passthrough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * PV guests under Xen are running in an non-contiguous memory architecture.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * When PCI pass-through is utilized, this necessitates an IOMMU for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * translating bus (DMA) to virtual and vice-versa and also providing a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * mechanism to have contiguous pages for device drivers operations (say DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * operations).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Specifically, under Xen the Linux idea of pages is an illusion. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * assumes that pages start at zero and go up to the available memory. To
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * help with that, the Linux Xen MMU provides a lookup mechanism to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * translate the page frame numbers (PFN) to machine frame numbers (MFN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * and vice-versa. The MFN are the "real" frame numbers. Furthermore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * memory is not contiguous. Xen hypervisor stitches memory for guests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * from different pools, which means there is no guarantee that PFN==MFN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * and PFN+1==MFN+1. Lastly with Xen 4.0, pages (in debug mode) are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * allocated in descending order (high to low), meaning the guest might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * never get any MFN's under the 4GB mark.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/dma-direct.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/dma-map-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <xen/swiotlb-xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <xen/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <xen/xen-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <xen/hvc-console.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/xen/page-coherent.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <trace/events/swiotlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define MAX_DMA_BITS 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * Used to do a quick range check in swiotlb_tbl_unmap_single and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * API.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static char *xen_io_tlb_start, *xen_io_tlb_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static unsigned long xen_io_tlb_nslabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Quick lookup value of the bus address of the IOTLB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) phys_addr_t baddr = (phys_addr_t)bfn << XEN_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) baddr |= paddr & ~XEN_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return baddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return phys_to_dma(dev, xen_phys_to_bus(dev, paddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static inline phys_addr_t xen_bus_to_phys(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) phys_addr_t baddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) phys_addr_t paddr = (xen_pfn << XEN_PAGE_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) (baddr & ~XEN_PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static inline phys_addr_t xen_dma_to_phys(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) dma_addr_t dma_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return xen_phys_to_dma(dev, virt_to_phys(address));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static inline int range_straddles_page_boundary(phys_addr_t p, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) unsigned long next_bfn, xen_pfn = XEN_PFN_DOWN(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned int i, nr_pages = XEN_PFN_UP(xen_offset_in_page(p) + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) next_bfn = pfn_to_bfn(xen_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) for (i = 1; i < nr_pages; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (pfn_to_bfn(++xen_pfn) != ++next_bfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned long xen_pfn = bfn_to_local_pfn(bfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /* If the address is outside our domain, it CAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * have the same virtual address as another address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * in our domain. Therefore _only_ check address within our domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (pfn_valid(PFN_DOWN(paddr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return paddr >= virt_to_phys(xen_io_tlb_start) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) paddr < virt_to_phys(xen_io_tlb_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) int i, rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) int dma_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) dma_addr_t dma_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) phys_addr_t p = virt_to_phys(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int slabs = min(nslabs - i, (unsigned long)IO_TLB_SEGSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) rc = xen_create_contiguous_region(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) p + (i << IO_TLB_SHIFT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) get_order(slabs << IO_TLB_SHIFT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) dma_bits, &dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) } while (rc && dma_bits++ < MAX_DMA_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) i += slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) } while (i < nslabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static unsigned long xen_set_nslabs(unsigned long nr_tbl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (!nr_tbl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) xen_io_tlb_nslabs = nr_tbl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return xen_io_tlb_nslabs << IO_TLB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) enum xen_swiotlb_err {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) XEN_SWIOTLB_UNKNOWN = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) XEN_SWIOTLB_ENOMEM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) XEN_SWIOTLB_EFIXUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) switch (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) case XEN_SWIOTLB_ENOMEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return "Cannot allocate Xen-SWIOTLB buffer\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) case XEN_SWIOTLB_EFIXUP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return "Failed to get contiguous memory for DMA from Xen!\n"\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) "You either: don't have the permissions, do not have"\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) " enough free memory under 4GB, or the hypervisor memory"\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) " is too fragmented!";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) int __ref xen_swiotlb_init(int verbose, bool early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) unsigned long bytes, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) int rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) unsigned int repeat = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) xen_io_tlb_nslabs = swiotlb_nr_tbl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) bytes = xen_set_nslabs(xen_io_tlb_nslabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * IO TLB memory already allocated. Just use it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (io_tlb_start != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) xen_io_tlb_start = phys_to_virt(io_tlb_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * Get IO TLB memory from any location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (early) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) xen_io_tlb_start = memblock_alloc(PAGE_ALIGN(bytes),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (!xen_io_tlb_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) __func__, PAGE_ALIGN(bytes), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) xen_io_tlb_start = (void *)xen_get_swiotlb_free_pages(order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (xen_io_tlb_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) order--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (order != get_order(bytes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) (PAGE_SIZE << order) >> 20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) xen_io_tlb_nslabs = SLABS_PER_PAGE << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (!xen_io_tlb_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) m_ret = XEN_SWIOTLB_ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * And replace that memory with pages under 4GB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) rc = xen_swiotlb_fixup(xen_io_tlb_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) xen_io_tlb_nslabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) memblock_free(__pa(xen_io_tlb_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) PAGE_ALIGN(bytes));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) free_pages((unsigned long)xen_io_tlb_start, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) xen_io_tlb_start = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) m_ret = XEN_SWIOTLB_EFIXUP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (early) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) verbose))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) panic("Cannot allocate SWIOTLB buffer");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) xen_io_tlb_end = xen_io_tlb_start + bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (!rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) swiotlb_set_max_segment(PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (repeat--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) (xen_io_tlb_nslabs >> 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) pr_info("Lowering to %luMB\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (early)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) free_pages((unsigned long)xen_io_tlb_start, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) dma_addr_t *dma_handle, gfp_t flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) int order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) u64 dma_mask = DMA_BIT_MASK(32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) phys_addr_t phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) dma_addr_t dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * Ignore region specifiers - the kernel's ideas of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * pseudo-phys memory layout has nothing to do with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * machine physical layout. We can't allocate highmem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * because we can't return a pointer to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) flags &= ~(__GFP_DMA | __GFP_HIGHMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /* Convert the size to actually allocated. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) size = 1UL << (order + XEN_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /* On ARM this function returns an ioremap'ped virtual address for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * which virt_to_phys doesn't return the corresponding physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * address. In fact on ARM virt_to_phys only works for kernel direct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * mapped RAM memory. Also see comment below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) ret = xen_alloc_coherent_pages(hwdev, size, dma_handle, flags, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (hwdev && hwdev->coherent_dma_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) dma_mask = hwdev->coherent_dma_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /* At this point dma_handle is the dma address, next we are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * going to set it to the machine address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Do not use virt_to_phys(ret) because on ARM it doesn't correspond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * to *dma_handle. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) phys = dma_to_phys(hwdev, *dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) dev_addr = xen_phys_to_dma(hwdev, phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (((dev_addr + size - 1 <= dma_mask)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) !range_straddles_page_boundary(phys, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) *dma_handle = dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (xen_create_contiguous_region(phys, order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) fls64(dma_mask), dma_handle) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) *dma_handle = phys_to_dma(hwdev, *dma_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) SetPageXenRemapped(virt_to_page(ret));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) memset(ret, 0, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) dma_addr_t dev_addr, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) phys_addr_t phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) u64 dma_mask = DMA_BIT_MASK(32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (hwdev && hwdev->coherent_dma_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) dma_mask = hwdev->coherent_dma_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) /* do not use virt_to_phys because on ARM it doesn't return you the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * physical address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) phys = xen_dma_to_phys(hwdev, dev_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /* Convert the size to actually allocated. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) size = 1UL << (order + XEN_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (is_vmalloc_addr(vaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) page = vmalloc_to_page(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) page = virt_to_page(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (!WARN_ON((dev_addr + size - 1 > dma_mask) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) range_straddles_page_boundary(phys, size)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) TestClearPageXenRemapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) xen_destroy_contiguous_region(phys, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) xen_free_coherent_pages(hwdev, size, vaddr, phys_to_dma(hwdev, phys),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * Map a single buffer of the indicated size for DMA in streaming mode. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * physical address to use is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * Once the device is given the dma address, the device owns this memory until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * either xen_swiotlb_unmap_page or xen_swiotlb_dma_sync_single is performed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) unsigned long offset, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) enum dma_data_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) phys_addr_t map, phys = page_to_phys(page) + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) dma_addr_t dev_addr = xen_phys_to_dma(dev, phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * If the address happens to be in the device's DMA window,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * we can safely return the device addr and not worry about bounce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * buffering it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (dma_capable(dev, dev_addr, size, true) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) !range_straddles_page_boundary(phys, size) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) !xen_arch_need_swiotlb(dev, phys, dev_addr) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) swiotlb_force != SWIOTLB_FORCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * Oh well, have to allocate and map a bounce buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) map = swiotlb_tbl_map_single(dev, phys, size, size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (map == (phys_addr_t)DMA_MAPPING_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) phys = map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) dev_addr = xen_phys_to_dma(dev, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * Ensure that the address returned is DMA'ble
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) swiotlb_tbl_unmap_single(dev, map, size, size, dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) attrs | DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) arch_sync_dma_for_device(phys, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) xen_dma_sync_for_device(dev, dev_addr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return dev_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Unmap a single streaming mode DMA translation. The dma_addr and size must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * match what was provided for in a previous xen_swiotlb_map_page call. All
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * other usages are undefined.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * After this call, reads by the cpu to the buffer are guaranteed to see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * whatever the device wrote there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) size_t size, enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) arch_sync_dma_for_cpu(paddr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /* NOTE: We use dev_addr here, not paddr! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (is_xen_swiotlb_buffer(hwdev, dev_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) size_t size, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (!dev_is_dma_coherent(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) arch_sync_dma_for_cpu(paddr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) xen_dma_sync_for_cpu(dev, dma_addr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (is_xen_swiotlb_buffer(dev, dma_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) size_t size, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (is_xen_swiotlb_buffer(dev, dma_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (!dev_is_dma_coherent(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) arch_sync_dma_for_device(paddr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) xen_dma_sync_for_device(dev, dma_addr, size, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * Unmap a set of streaming mode DMA translations. Again, cpu read rules
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * concerning calls here are the same as for swiotlb_unmap_page() above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) for_each_sg(sgl, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) xen_swiotlb_unmap_page(hwdev, sg->dma_address, sg_dma_len(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) xen_swiotlb_map_sg(struct device *dev, struct scatterlist *sgl, int nelems,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) for_each_sg(sgl, sg, nelems, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) sg->dma_address = xen_swiotlb_map_page(dev, sg_page(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) sg->offset, sg->length, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (sg->dma_address == DMA_MAPPING_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto out_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) sg_dma_len(sg) = sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return nelems;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) out_unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) xen_swiotlb_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) sg_dma_len(sgl) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) xen_swiotlb_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) int nelems, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) for_each_sg(sgl, sg, nelems, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) xen_swiotlb_sync_single_for_cpu(dev, sg->dma_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) sg->length, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) int nelems, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) for_each_sg(sgl, sg, nelems, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) xen_swiotlb_sync_single_for_device(dev, sg->dma_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) sg->length, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Return whether the given device DMA address mask can be supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * properly. For example, if your device can only drive the low 24-bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * during bus mastering, then you would pass 0x00ffffff as the mask to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) const struct dma_map_ops xen_swiotlb_dma_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) .alloc = xen_swiotlb_alloc_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) .free = xen_swiotlb_free_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) .sync_single_for_cpu = xen_swiotlb_sync_single_for_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) .sync_single_for_device = xen_swiotlb_sync_single_for_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) .sync_sg_for_cpu = xen_swiotlb_sync_sg_for_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) .sync_sg_for_device = xen_swiotlb_sync_sg_for_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) .map_sg = xen_swiotlb_map_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) .unmap_sg = xen_swiotlb_unmap_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) .map_page = xen_swiotlb_map_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) .unmap_page = xen_swiotlb_unmap_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) .dma_supported = xen_swiotlb_dma_supported,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) .mmap = dma_common_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) .get_sgtable = dma_common_get_sgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .alloc_pages = dma_common_alloc_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .free_pages = dma_common_free_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags