^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright © 2006-2014 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Authors: David Woodhouse <dwmw2@infradead.org>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Ashok Raj <ashok.raj@intel.com>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Shaohua Li <shaohua.li@intel.com>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Fenghua Yu <fenghua.yu@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Joerg Roedel <jroedel@suse.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define pr_fmt(fmt) "DMAR: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define dev_fmt(fmt) pr_fmt(fmt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/bitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/dmar.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/dma-map-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/timer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/iova.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/intel-iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/syscore_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/tboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/dmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/pci-ats.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/dma-map-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/dma-direct.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/crash_dump.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/numa.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/swiotlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/irq_remapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <asm/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <trace/events/intel_iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include "../irq_remapping.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include "pasid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define ROOT_SIZE VTD_PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define CONTEXT_SIZE VTD_PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define IOAPIC_RANGE_START (0xfee00000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define IOAPIC_RANGE_END (0xfeefffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define IOVA_START_ADDR (0x1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define MAX_AGAW_WIDTH 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) to match. That way, we can use 'unsigned long' for PFNs with impunity. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /* IO virtual address start page frame number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define IOVA_START_PFN (1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* page table handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define LEVEL_STRIDE (9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * This bitmap is used to advertise the page sizes our hardware support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * to the IOMMU core, which will then use this information to split
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * physically contiguous memory regions it is mapping into page sizes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * that we support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Traditionally the IOMMU core just handed us the mappings directly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * after making sure the size is an order of a 4KiB page and that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * mapping has natural alignment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * To retain this behavior, we currently advertise that we support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * all page sizes that are an order of 4KiB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * If at some point we'd like to utilize the IOMMU core's new behavior,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * we could change this to advertise the real page sizes we support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static inline int agaw_to_level(int agaw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return agaw + 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) static inline int agaw_to_width(int agaw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static inline int width_to_agaw(int width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static inline unsigned int level_to_offset_bits(int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return (level - 1) * LEVEL_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static inline int pfn_level_offset(u64 pfn, int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static inline u64 level_mask(int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return -1ULL << level_to_offset_bits(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static inline u64 level_size(int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return 1ULL << level_to_offset_bits(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static inline u64 align_to_level(u64 pfn, int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return (pfn + level_size(level) - 1) & level_mask(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) are never going to work. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static inline unsigned long page_to_dma_pfn(struct page *pg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return mm_to_dma_pfn(page_to_pfn(pg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static inline unsigned long virt_to_dma_pfn(void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) return page_to_dma_pfn(virt_to_page(p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /* global iommu list, set NULL for ignored DMAR units */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) static struct intel_iommu **g_iommus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static void __init check_tylersburg_isoch(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static int rwbf_quirk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * set to 1 to panic kernel if can't successfully enable VT-d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * (used when kernel is launched w/ TXT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) static int force_on = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) static int intel_iommu_tboot_noforce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static int no_platform_optin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * Take a root_entry and return the Lower Context Table Pointer (LCTP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * if marked present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static phys_addr_t root_entry_lctp(struct root_entry *re)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (!(re->lo & 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return re->lo & VTD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * Take a root_entry and return the Upper Context Table Pointer (UCTP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * if marked present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static phys_addr_t root_entry_uctp(struct root_entry *re)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (!(re->hi & 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return re->hi & VTD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static inline void context_clear_pasid_enable(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) context->lo &= ~(1ULL << 11);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static inline bool context_pasid_enabled(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return !!(context->lo & (1ULL << 11));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static inline void context_set_copied(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) context->hi |= (1ull << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static inline bool context_copied(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return !!(context->hi & (1ULL << 3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static inline bool __context_present(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return (context->lo & 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) bool context_present(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return context_pasid_enabled(context) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) __context_present(context) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) __context_present(context) && !context_copied(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static inline void context_set_present(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) context->lo |= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static inline void context_set_fault_enable(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) context->lo &= (((u64)-1) << 2) | 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static inline void context_set_translation_type(struct context_entry *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) context->lo &= (((u64)-1) << 4) | 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) context->lo |= (value & 3) << 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static inline void context_set_address_root(struct context_entry *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) context->lo &= ~VTD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) context->lo |= value & VTD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static inline void context_set_address_width(struct context_entry *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) context->hi |= value & 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static inline void context_set_domain_id(struct context_entry *context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) context->hi |= (value & ((1 << 16) - 1)) << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static inline int context_domain_id(struct context_entry *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return((c->hi >> 8) & 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static inline void context_clear_entry(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) context->lo = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) context->hi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * This domain is a statically identity mapping domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * 1. This domain creats a static 1:1 mapping to all usable memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * 2. It maps to each iommu if successful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * 3. Each iommu mapps to this domain if successful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static struct dmar_domain *si_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static int hw_pass_through = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #define for_each_domain_iommu(idx, domain) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) for (idx = 0; idx < g_num_of_iommus; idx++) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (domain->iommu_refcnt[idx])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct dmar_rmrr_unit {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) struct list_head list; /* list of rmrr units */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) struct acpi_dmar_header *hdr; /* ACPI header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) u64 base_address; /* reserved base address*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) u64 end_address; /* reserved end address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct dmar_dev_scope *devices; /* target devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int devices_cnt; /* target device count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) struct dmar_atsr_unit {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) struct list_head list; /* list of ATSR units */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct acpi_dmar_header *hdr; /* ACPI header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct dmar_dev_scope *devices; /* target devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) int devices_cnt; /* target device count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) u8 include_all:1; /* include all ports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static LIST_HEAD(dmar_atsr_units);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static LIST_HEAD(dmar_rmrr_units);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) #define for_each_rmrr_units(rmrr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) list_for_each_entry(rmrr, &dmar_rmrr_units, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* bitmap for indexing intel_iommus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static int g_num_of_iommus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static void domain_exit(struct dmar_domain *domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static void domain_remove_dev_info(struct dmar_domain *domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static void dmar_remove_one_dev_info(struct device *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static void __dmar_remove_one_dev_info(struct device_domain_info *info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static int intel_iommu_attach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct device *dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) dma_addr_t iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) int dmar_disabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) int dmar_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) #endif /* CONFIG_INTEL_IOMMU_DEFAULT_ON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) #ifdef CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) int intel_iommu_sm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int intel_iommu_sm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #endif /* CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) int intel_iommu_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) EXPORT_SYMBOL_GPL(intel_iommu_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static int dmar_map_gfx = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static int dmar_forcedac;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static int intel_iommu_strict;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int intel_iommu_superpage = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static int iommu_identity_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static int intel_no_bounce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) static int iommu_skip_te_disable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) #define IDENTMAP_GFX 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #define IDENTMAP_AZALIA 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) int intel_iommu_gfx_mapped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct device_domain_info *get_domain_info(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) info = dev_iommu_priv_get(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) DEFINE_SPINLOCK(device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static LIST_HEAD(device_domain_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) to_pci_dev(d)->untrusted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * Iterate over elements in device_domain_list and call the specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * callback @fn against each element.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) int for_each_device_domain(int (*fn)(struct device_domain_info *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void *data), void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) list_for_each_entry(info, &device_domain_list, global) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) ret = fn(info, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) const struct iommu_ops intel_iommu_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static bool translation_pre_enabled(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) static void clear_translation_pre_enabled(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static void init_translation_status(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) u32 gsts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) gsts = readl(iommu->reg + DMAR_GSTS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (gsts & DMA_GSTS_TES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static int __init intel_iommu_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (!str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) while (*str) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (!strncmp(str, "on", 2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) dmar_disabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) pr_info("IOMMU enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) } else if (!strncmp(str, "off", 3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) dmar_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) no_platform_optin = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) pr_info("IOMMU disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) } else if (!strncmp(str, "igfx_off", 8)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) dmar_map_gfx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) pr_info("Disable GFX device mapping\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) } else if (!strncmp(str, "forcedac", 8)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) pr_info("Forcing DAC for PCI devices\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) dmar_forcedac = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) } else if (!strncmp(str, "strict", 6)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) pr_info("Disable batched IOTLB flush\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) intel_iommu_strict = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) } else if (!strncmp(str, "sp_off", 6)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) pr_info("Disable supported super page\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) intel_iommu_superpage = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) } else if (!strncmp(str, "sm_on", 5)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) pr_info("Intel-IOMMU: scalable mode supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) intel_iommu_sm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) } else if (!strncmp(str, "tboot_noforce", 13)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) intel_iommu_tboot_noforce = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) } else if (!strncmp(str, "nobounce", 8)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) pr_info("Intel-IOMMU: No bounce buffer. This could expose security risks of DMA attacks\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) intel_no_bounce = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) str += strcspn(str, ",");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) while (*str == ',')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) str++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) __setup("intel_iommu=", intel_iommu_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) static struct kmem_cache *iommu_domain_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static struct kmem_cache *iommu_devinfo_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct dmar_domain **domains;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) int idx = did >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) domains = iommu->domains[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (!domains)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return domains[did & 0xff];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) struct dmar_domain **domains;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) int idx = did >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (!iommu->domains[idx]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) size_t size = 256 * sizeof(struct dmar_domain *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) domains = iommu->domains[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (WARN_ON(!domains))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) domains[did & 0xff] = domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) void *alloc_pgtable_page(int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) void *vaddr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) vaddr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) void free_pgtable_page(void *vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) free_page((unsigned long)vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static inline void *alloc_domain_mem(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) static void free_domain_mem(void *vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) kmem_cache_free(iommu_domain_cache, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static inline void * alloc_devinfo_mem(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) static inline void free_devinfo_mem(void *vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) kmem_cache_free(iommu_devinfo_cache, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static inline int domain_type_is_si(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static inline bool domain_use_first_level(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) static inline int domain_pfn_supported(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) unsigned long sagaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) int agaw = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) sagaw = cap_sagaw(iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) for (agaw = width_to_agaw(max_gaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) agaw >= 0; agaw--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (test_bit(agaw, &sagaw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * Calculate max SAGAW for each iommu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * calculate agaw for each iommu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * "SAGAW" may be different across iommus, use a default agaw, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * get a supported less agaw for iommus that don't support the default agaw.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) int iommu_calculate_agaw(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /* This functionin only returns single iommu in a domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) int iommu_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) /* si_domain and vm domain should not get here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (WARN_ON(domain->domain.type != IOMMU_DOMAIN_DMA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) for_each_domain_iommu(iommu_id, domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return g_iommus[iommu_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return sm_supported(iommu) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static void domain_update_iommu_coherency(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) domain->iommu_coherency = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) for_each_domain_iommu(i, domain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) found = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (!iommu_paging_structure_coherency(g_iommus[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) domain->iommu_coherency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (found)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) /* No hardware attached; use lowest common denominator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (!iommu_paging_structure_coherency(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) domain->iommu_coherency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) static int domain_update_iommu_snooping(struct intel_iommu *skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) int ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (iommu != skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * If the hardware is operating in the scalable mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * the snooping control is always supported since we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * always set PASID-table-entry.PGSNP bit if the domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * is managed outside (UNMANAGED).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (!sm_supported(iommu) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) !ecap_sc_support(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) static int domain_update_iommu_superpage(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) struct intel_iommu *skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) int mask = 0x3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (!intel_iommu_superpage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* set iommu_superpage to the smallest common denominator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) if (iommu != skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (domain && domain_use_first_level(domain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) if (!cap_fl1gp_support(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) mask = 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) mask &= cap_super_page_val(iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) if (!mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) return fls(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static int domain_update_device_node(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) int nid = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) if (list_empty(&domain->devices))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) return NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) list_for_each_entry(info, &domain->devices, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (!info->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * There could possibly be multiple device numa nodes as devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * within the same domain may sit behind different IOMMUs. There
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * isn't perfect answer in such situation, so we select first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * come first served policy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) nid = dev_to_node(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (nid != NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) return nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) /* Some capabilities may be different across iommus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) static void domain_update_iommu_cap(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) domain_update_iommu_coherency(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) domain->iommu_snooping = domain_update_iommu_snooping(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * If RHSA is missing, we should default to the device numa domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * as fall back.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (domain->nid == NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) domain->nid = domain_update_device_node(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * First-level translation restricts the input-address to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * canonical address (i.e., address bits 63:N have the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * value as address bit [N-1], where N is 48-bits with 4-level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * paging and 57-bits with 5-level paging). Hence, skip bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * [N-1].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) u8 devfn, int alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) struct root_entry *root = &iommu->root_entry[bus];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) u64 *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) entry = &root->lo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (devfn >= 0x80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) devfn -= 0x80;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) entry = &root->hi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) devfn *= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (*entry & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) context = phys_to_virt(*entry & VTD_PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) unsigned long phy_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) if (!alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) context = alloc_pgtable_page(iommu->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (!context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) phy_addr = virt_to_phys((void *)context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) *entry = phy_addr | 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) __iommu_flush_cache(iommu, entry, sizeof(*entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) return &context[devfn];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) static bool attach_deferred(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * is_downstream_to_pci_bridge - test if a device belongs to the PCI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * sub-hierarchy of a candidate PCI-PCI bridge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * @bridge: the candidate PCI-PCI bridge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct pci_dev *pdev, *pbridge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (!dev_is_pci(dev) || !dev_is_pci(bridge))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) pbridge = to_pci_dev(bridge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (pbridge->subordinate &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) pbridge->subordinate->number <= pdev->bus->number &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) pbridge->subordinate->busn_res.end >= pdev->bus->number)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) u32 vtbar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /* We know that this device on this chipset has its own IOMMU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * If we find it under a different IOMMU, then the BIOS is lying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * to us. Hope that the IOMMU for this device is actually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * disabled, and it needs no translation...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) /* "can't" happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) dev_info(&pdev->dev, "failed to run vt-d quirk\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) vtbar &= 0xffff0000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) /* we know that the this iommu should be at offset 0xa000 from vtbar */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) drhd = dmar_find_matched_drhd_unit(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (!iommu || iommu->drhd->ignored)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) if (dev_is_pci(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) quirk_ioat_snb_local_iommu(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) struct dmar_drhd_unit *drhd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) struct pci_dev *pdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) struct device *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) u16 segment = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) if (dev_is_pci(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) struct pci_dev *pf_pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) pdev = pci_real_dma_dev(to_pci_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) /* VFs aren't listed in scope tables; we need to look up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * the PF instead to find the IOMMU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) pf_pdev = pci_physfn(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) dev = &pf_pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) segment = pci_domain_nr(pdev->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) } else if (has_acpi_companion(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) dev = &ACPI_COMPANION(dev)->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) for_each_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (pdev && segment != drhd->segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) for_each_active_dev_scope(drhd->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) drhd->devices_cnt, i, tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (tmp == dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) /* For a VF use its original BDF# not that of the PF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * which we used for the IOMMU lookup. Strictly speaking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * we could do this for all PCI devices; we only need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * get the BDF# from the scope table for ACPI matches. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (pdev && pdev->is_virtfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) goto got_pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (bus && devfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) *bus = drhd->devices[i].bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) *devfn = drhd->devices[i].devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) if (is_downstream_to_pci_bridge(dev, tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) goto got_pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) if (pdev && drhd->include_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) got_pdev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) if (bus && devfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) *bus = pdev->bus->number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) *devfn = pdev->devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (iommu_is_dummy(iommu, dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) return iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) static void domain_flush_cache(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) void *addr, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) if (!domain->iommu_coherency)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) clflush_cache_range(addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) context = iommu_context_addr(iommu, bus, devfn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) if (context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) ret = context_present(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) static void free_context_table(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (!iommu->root_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) for (i = 0; i < ROOT_ENTRY_NR; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) context = iommu_context_addr(iommu, i, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) free_pgtable_page(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (!sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) context = iommu_context_addr(iommu, i, 0x80, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) free_pgtable_page(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) free_pgtable_page(iommu->root_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) iommu->root_entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) unsigned long pfn, int *target_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) struct dma_pte *parent, *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) int level = agaw_to_level(domain->agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) BUG_ON(!domain->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (!domain_pfn_supported(domain, pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) /* Address beyond IOMMU's addressing capabilities. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) parent = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) void *tmp_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) offset = pfn_level_offset(pfn, level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) pte = &parent[offset];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) if (level == *target_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (!dma_pte_present(pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) uint64_t pteval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) tmp_page = alloc_pgtable_page(domain->nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) if (!tmp_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (domain_use_first_level(domain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) if (domain->domain.type == IOMMU_DOMAIN_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) pteval |= DMA_FL_PTE_ACCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (cmpxchg64(&pte->val, 0ULL, pteval))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) /* Someone else set it while we were thinking; use theirs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) free_pgtable_page(tmp_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) domain_flush_cache(domain, pte, sizeof(*pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (level == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) parent = phys_to_virt(dma_pte_addr(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) level--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (!*target_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) *target_level = level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) /* return address's pte at specific level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) unsigned long pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) int level, int *large_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) struct dma_pte *parent, *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) int total = agaw_to_level(domain->agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) parent = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) while (level <= total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) offset = pfn_level_offset(pfn, total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) pte = &parent[offset];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) if (level == total)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (!dma_pte_present(pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) *large_page = total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) if (dma_pte_superpage(pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) *large_page = total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) parent = phys_to_virt(dma_pte_addr(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) total--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /* clear last level pte, a tlb flush should be followed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) static void dma_pte_clear_range(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) unsigned long last_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) unsigned int large_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) struct dma_pte *first_pte, *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) BUG_ON(!domain_pfn_supported(domain, start_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) BUG_ON(!domain_pfn_supported(domain, last_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) BUG_ON(start_pfn > last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) /* we don't need lock here; nobody else touches the iova range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) large_page = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (!pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) start_pfn = align_to_level(start_pfn + 1, large_page + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) dma_clear_pte(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) start_pfn += lvl_to_nr_pages(large_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) pte++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) domain_flush_cache(domain, first_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) (void *)pte - (void *)first_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) } while (start_pfn && start_pfn <= last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) static void dma_pte_free_level(struct dmar_domain *domain, int level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) int retain_level, struct dma_pte *pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) unsigned long pfn, unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) unsigned long last_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) pfn = max(start_pfn, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) pte = &pte[pfn_level_offset(pfn, level)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) unsigned long level_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) struct dma_pte *level_pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (!dma_pte_present(pte) || dma_pte_superpage(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) level_pfn = pfn & level_mask(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) level_pte = phys_to_virt(dma_pte_addr(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (level > 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) dma_pte_free_level(domain, level - 1, retain_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) level_pte, level_pfn, start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * Free the page table if we're below the level we want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * retain and the range covers the entire table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) if (level < retain_level && !(start_pfn > level_pfn ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) last_pfn < level_pfn + level_size(level) - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) dma_clear_pte(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) domain_flush_cache(domain, pte, sizeof(*pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) free_pgtable_page(level_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) pfn += level_size(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) * clear last level (leaf) ptes and free page table pages below the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) * level we wish to keep intact.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) static void dma_pte_free_pagetable(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) unsigned long last_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) int retain_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) BUG_ON(!domain_pfn_supported(domain, start_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) BUG_ON(!domain_pfn_supported(domain, last_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) BUG_ON(start_pfn > last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) dma_pte_clear_range(domain, start_pfn, last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) /* We don't need lock here; nobody else touches the iova range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) domain->pgd, 0, start_pfn, last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) /* free pgd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) free_pgtable_page(domain->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) domain->pgd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) /* When a page at a given level is being unlinked from its parent, we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) need to *modify* it at all. All we need to do is make a list of all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) pages which can be freed just as soon as we've flushed the IOTLB and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) know the hardware page-walk will no longer touch them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) The 'pte' argument is the *parent* PTE, pointing to the page that is to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) be freed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) int level, struct dma_pte *pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) struct page *freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) struct page *pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) pg->freelist = freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) freelist = pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (level == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) pte = page_address(pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) if (dma_pte_present(pte) && !dma_pte_superpage(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) freelist = dma_pte_list_pagetables(domain, level - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) pte, freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) pte++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) } while (!first_pte_in_page(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) struct dma_pte *pte, unsigned long pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) unsigned long last_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) struct page *freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) struct dma_pte *first_pte = NULL, *last_pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) pfn = max(start_pfn, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) pte = &pte[pfn_level_offset(pfn, level)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) unsigned long level_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (!dma_pte_present(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) level_pfn = pfn & level_mask(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /* If range covers entire pagetable, free it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if (start_pfn <= level_pfn &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) last_pfn >= level_pfn + level_size(level) - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) /* These suborbinate page tables are going away entirely. Don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) bother to clear them; we're just going to *free* them. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) if (level > 1 && !dma_pte_superpage(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) dma_clear_pte(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) if (!first_pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) first_pte = pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) last_pte = pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) } else if (level > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) /* Recurse down into a level that isn't *entirely* obsolete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) freelist = dma_pte_clear_level(domain, level - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) phys_to_virt(dma_pte_addr(pte)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) level_pfn, start_pfn, last_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) pfn += level_size(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (first_pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) domain_flush_cache(domain, first_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) (void *)++last_pte - (void *)first_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /* We can't just free the pages because the IOMMU may still be walking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) the page tables, and may have cached the intermediate levels. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) pages can only be freed after the IOTLB flush has been done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) static struct page *domain_unmap(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) unsigned long last_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) struct page *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) BUG_ON(!domain_pfn_supported(domain, start_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) BUG_ON(!domain_pfn_supported(domain, last_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) BUG_ON(start_pfn > last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) /* we don't need lock here; nobody else touches the iova range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) domain->pgd, 0, start_pfn, last_pfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /* free pgd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) struct page *pgd_page = virt_to_page(domain->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) pgd_page->freelist = freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) freelist = pgd_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) domain->pgd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) static void dma_free_pagelist(struct page *freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) struct page *pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) while ((pg = freelist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) freelist = pg->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) free_pgtable_page(page_address(pg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) static void iova_entry_free(unsigned long data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) struct page *freelist = (struct page *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) dma_free_pagelist(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) /* iommu handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) static int iommu_alloc_root_entry(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) struct root_entry *root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) root = (struct root_entry *)alloc_pgtable_page(iommu->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) if (!root) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) pr_err("Allocating root entry for %s failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) __iommu_flush_cache(iommu, root, ROOT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) iommu->root_entry = root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) static void iommu_set_root_entry(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) u32 sts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) addr = virt_to_phys(iommu->root_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) addr |= DMA_RTADDR_SMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) readl, (sts & DMA_GSTS_RTPS), sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) if (sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) void iommu_flush_write_buffer(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) if (!rwbf_quirk && !cap_rwbf(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) readl, (!(val & DMA_GSTS_WBFS)), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) /* return value determine if we need a write buffer flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) static void __iommu_flush_context(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) u16 did, u16 source_id, u8 function_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) u64 type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) u64 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) case DMA_CCMD_GLOBAL_INVL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) val = DMA_CCMD_GLOBAL_INVL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) case DMA_CCMD_DOMAIN_INVL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) case DMA_CCMD_DEVICE_INVL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) val |= DMA_CCMD_ICC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) dmar_readq, (!(val & DMA_CCMD_ICC)), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) /* return value determine if we need a write buffer flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) u64 addr, unsigned int size_order, u64 type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) int tlb_offset = ecap_iotlb_offset(iommu->ecap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) u64 val = 0, val_iva = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) case DMA_TLB_GLOBAL_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) /* global flush doesn't need set IVA_REG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) case DMA_TLB_DSI_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) case DMA_TLB_PSI_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) /* IH bit is passed in as part of address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) val_iva = size_order | addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) /* Note: set drain read/write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) #if 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) * This is probably to be super secure.. Looks like we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) * ignore it without any impact.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) if (cap_read_drain(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) val |= DMA_TLB_READ_DRAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) if (cap_write_drain(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) val |= DMA_TLB_WRITE_DRAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) /* Note: Only uses first TLB reg currently */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (val_iva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) dmar_writeq(iommu->reg + tlb_offset, val_iva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) dmar_writeq(iommu->reg + tlb_offset + 8, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) IOMMU_WAIT_OP(iommu, tlb_offset + 8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) dmar_readq, (!(val & DMA_TLB_IVT)), val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) /* check IOTLB invalidation granularity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (DMA_TLB_IAIG(val) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) pr_err("Flush IOTLB failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) pr_debug("TLB flush request %Lx, actual %Lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) (unsigned long long)DMA_TLB_IIRG(type),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) (unsigned long long)DMA_TLB_IAIG(val));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) static struct device_domain_info *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) u8 bus, u8 devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) if (!iommu->qi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) list_for_each_entry(info, &domain->devices, link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) if (info->iommu == iommu && info->bus == bus &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) info->devfn == devfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) if (info->ats_supported && info->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) static void domain_update_iotlb(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) bool has_iotlb_device = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) list_for_each_entry(info, &domain->devices, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) if (!info->dev || !dev_is_pci(info->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) pdev = to_pci_dev(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) if (pdev->ats_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) has_iotlb_device = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) domain->has_iotlb_device = has_iotlb_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) static void iommu_enable_dev_iotlb(struct device_domain_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (!info || !dev_is_pci(info->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) pdev = to_pci_dev(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) /* For IOMMU that supports device IOTLB throttling (DIT), we assign
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) * queue depth at PF level. If DIT is not set, PFSID will be treated as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) * reserved, which should be set to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) if (!ecap_dit(info->iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) info->pfsid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) struct pci_dev *pf_pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) /* pdev will be returned if device is not a vf */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) pf_pdev = pci_physfn(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) info->pfsid = pci_dev_id(pf_pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) /* The PCIe spec, in its wisdom, declares that the behaviour of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) the device if you enable PASID support after ATS support is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) undefined. So always enable PASID support on devices which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) have it, even if we can't yet know if we're ever going to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) use it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) info->pasid_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (info->pri_supported &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) info->pri_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) if (info->ats_supported && pci_ats_page_aligned(pdev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) info->ats_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) domain_update_iotlb(info->domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) info->ats_qdep = pci_ats_queue_depth(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) static void iommu_disable_dev_iotlb(struct device_domain_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) if (!dev_is_pci(info->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) pdev = to_pci_dev(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) if (info->ats_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) pci_disable_ats(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) info->ats_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) domain_update_iotlb(info->domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (info->pri_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) pci_disable_pri(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) info->pri_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) if (info->pasid_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) pci_disable_pasid(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) info->pasid_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) u64 addr, unsigned mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) u16 sid, qdep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) if (!domain->has_iotlb_device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) list_for_each_entry(info, &domain->devices, link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) if (!info->ats_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) sid = info->bus << 8 | info->devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) qdep = info->ats_qdep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) qdep, addr, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) static void domain_flush_piotlb(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) u64 addr, unsigned long npages, bool ih)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) u16 did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (domain->default_pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) qi_flush_piotlb(iommu, did, domain->default_pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) addr, npages, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) if (!list_empty(&domain->devices))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, npages, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) unsigned long pfn, unsigned int pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) int ih, int map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) unsigned int mask = ilog2(__roundup_pow_of_two(pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) u16 did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) BUG_ON(pages == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) if (ih)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) ih = 1 << 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) if (domain_use_first_level(domain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) domain_flush_piotlb(iommu, domain, addr, pages, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) * Fallback to domain selective flush if no PSI support or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) * the size is too big. PSI requires page size to be 2 ^ x,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) * and the base address is naturally aligned to the size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) if (!cap_pgsel_inv(iommu->cap) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) mask > cap_max_amask_val(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) iommu->flush.flush_iotlb(iommu, did, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) DMA_TLB_PSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * In caching mode, changes of pages from non-present to present require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) * flush. However, device IOTLB doesn't need to be flushed in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) if (!cap_caching_mode(iommu->cap) || !map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) iommu_flush_dev_iotlb(domain, addr, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) /* Notification for newly created mappings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) static inline void __mapping_notify_one(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) unsigned long pfn, unsigned int pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) * It's a non-present to present mapping. Only flush if caching mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) * and second level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) static void iommu_flush_iova(struct iova_domain *iovad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) domain = container_of(iovad, struct dmar_domain, iovad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) for_each_domain_iommu(idx, domain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) struct intel_iommu *iommu = g_iommus[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) u16 did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) if (domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) domain_flush_piotlb(iommu, domain, 0, -1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) iommu->flush.flush_iotlb(iommu, did, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) if (!cap_caching_mode(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) 0, MAX_AGAW_PFN_WIDTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) u32 pmen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) raw_spin_lock_irqsave(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) pmen = readl(iommu->reg + DMAR_PMEN_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) pmen &= ~DMA_PMEN_EPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) writel(pmen, iommu->reg + DMAR_PMEN_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) /* wait for the protected region status bit to clear */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) readl, !(pmen & DMA_PMEN_PRS), pmen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) static void iommu_enable_translation(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) u32 sts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) raw_spin_lock_irqsave(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) iommu->gcmd |= DMA_GCMD_TE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) readl, (sts & DMA_GSTS_TES), sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) static void iommu_disable_translation(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) u32 sts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) iommu->gcmd &= ~DMA_GCMD_TE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) /* Make sure hardware complete it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) readl, (!(sts & DMA_GSTS_TES)), sts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) static int iommu_init_domains(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) u32 ndomains, nlongs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) ndomains = cap_ndoms(iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) pr_debug("%s: Number of Domains supported <%d>\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) iommu->name, ndomains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) nlongs = BITS_TO_LONGS(ndomains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) spin_lock_init(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) if (!iommu->domain_ids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) pr_err("%s: Allocating domain id array failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) iommu->domains = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) if (iommu->domains) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) size = 256 * sizeof(struct dmar_domain *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) iommu->domains[0] = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) if (!iommu->domains || !iommu->domains[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) pr_err("%s: Allocating domain array failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) kfree(iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) kfree(iommu->domains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) iommu->domain_ids = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) iommu->domains = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * If Caching mode is set, then invalid translations are tagged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * with domain-id 0, hence we need to pre-allocate it. We also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * use domain-id 0 as a marker for non-allocated domain-id, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * make sure it is not used for a real domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) set_bit(0, iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) * entry for first-level or pass-through translation modes should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) * be programmed with a domain id different from those used for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) * second-level or nested translation. We reserve a domain id for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) * this purpose.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) if (sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) static void disable_dmar_iommu(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) struct device_domain_info *info, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) if (!iommu->domains || !iommu->domain_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) if (info->iommu != iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) if (!info->dev || !info->domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) __dmar_remove_one_dev_info(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) if (iommu->gcmd & DMA_GCMD_TE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) static void free_dmar_iommu(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) if ((iommu->domains) && (iommu->domain_ids)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) for (i = 0; i < elems; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) kfree(iommu->domains[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) kfree(iommu->domains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) kfree(iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) iommu->domains = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) iommu->domain_ids = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) g_iommus[iommu->seq_id] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) /* free context mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) free_context_table(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) if (pasid_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) if (ecap_prs(iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) intel_svm_finish_prq(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) if (vccap_pasid(iommu->vccap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) ioasid_unregister_allocator(&iommu->pasid_allocator);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) * Check and return whether first level is used by default for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) * DMA translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) static bool first_level_by_default(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) static int first_level_support = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) if (likely(first_level_support != -1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) return first_level_support;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) first_level_support = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) if (!sm_supported(iommu) || !ecap_flts(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) first_level_support = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) return first_level_support;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) static struct dmar_domain *alloc_domain(int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) domain = alloc_domain_mem();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) if (!domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) memset(domain, 0, sizeof(*domain));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) domain->nid = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) domain->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) if (first_level_by_default())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) domain->has_iotlb_device = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) INIT_LIST_HEAD(&domain->devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) return domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) /* Must be called with iommu->lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) static int domain_attach_iommu(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) unsigned long ndomains;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) assert_spin_locked(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) domain->iommu_refcnt[iommu->seq_id] += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) domain->iommu_count += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) if (domain->iommu_refcnt[iommu->seq_id] == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) ndomains = cap_ndoms(iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) num = find_first_zero_bit(iommu->domain_ids, ndomains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) if (num >= ndomains) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) pr_err("%s: No free domain ids\n", iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) domain->iommu_refcnt[iommu->seq_id] -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) domain->iommu_count -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) set_bit(num, iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) set_iommu_domain(iommu, num, domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) domain->iommu_did[iommu->seq_id] = num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) domain->nid = iommu->node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) domain_update_iommu_cap(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) static int domain_detach_iommu(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) int num, count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) assert_spin_locked(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) domain->iommu_refcnt[iommu->seq_id] -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) count = --domain->iommu_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) if (domain->iommu_refcnt[iommu->seq_id] == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) num = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) clear_bit(num, iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) set_iommu_domain(iommu, num, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) domain_update_iommu_cap(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) domain->iommu_did[iommu->seq_id] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) static struct iova_domain reserved_iova_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) static struct lock_class_key reserved_rbtree_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) static int dmar_init_reserved_ranges(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) struct pci_dev *pdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) struct iova *iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) &reserved_rbtree_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) /* IOAPIC ranges shouldn't be accessed by DMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) IOVA_PFN(IOAPIC_RANGE_END));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) if (!iova) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) pr_err("Reserve IOAPIC range failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) /* Reserve all PCI MMIO to avoid peer-to-peer access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) for_each_pci_dev(pdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) struct resource *r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) for (i = 0; i < PCI_NUM_RESOURCES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) r = &pdev->resource[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) if (!r->flags || !(r->flags & IORESOURCE_MEM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) iova = reserve_iova(&reserved_iova_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) IOVA_PFN(r->start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) IOVA_PFN(r->end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) if (!iova) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) pci_err(pdev, "Reserve iova for %pR failed\n", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) static inline int guestwidth_to_adjustwidth(int gaw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) int agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) int r = (gaw - 12) % 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) if (r == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) agaw = gaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) agaw = gaw + 9 - r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) if (agaw > 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) agaw = 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) return agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) static void domain_exit(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) /* Remove associated devices and clear attached or cached domains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) domain_remove_dev_info(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) /* destroy iovas */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) if (domain->domain.type == IOMMU_DOMAIN_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) put_iova_domain(&domain->iovad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (domain->pgd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) struct page *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) dma_free_pagelist(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) free_domain_mem(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) * Get the PASID directory size for scalable mode context entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * Value of X in the PDTS field of a scalable mode context entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) * indicates PASID directory with 2^(X + 7) entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) static inline unsigned long context_get_sm_pds(struct pasid_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) int pds, max_pde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) max_pde = table->max_pasid >> PASID_PDE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) if (pds < 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) return pds - 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) * Set the RID_PASID field of a scalable mode context entry. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) * IOMMU hardware will use the PASID value set in this field for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) * DMA translations of DMA requests without PASID.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) context->hi |= pasid & ((1 << 20) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) * Set the DTE(Device-TLB Enable) field of a scalable mode context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) static inline void context_set_sm_dte(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) context->lo |= (1 << 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) * Set the PRE(Page Request Enable) field of a scalable mode context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) static inline void context_set_sm_pre(struct context_entry *context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) context->lo |= (1 << 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) /* Convert value to context PASID directory size field coding. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) #define context_pdts(pds) (((pds) & 0x7) << 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) static int domain_context_mapping_one(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) struct pasid_table *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) u8 bus, u8 devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) u16 did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) int translation = CONTEXT_TT_MULTI_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) struct device_domain_info *info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) WARN_ON(did == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) if (hw_pass_through && domain_type_is_si(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) translation = CONTEXT_TT_PASS_THROUGH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) pr_debug("Set context mapping for %02x:%02x.%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) BUG_ON(!domain->pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) context = iommu_context_addr(iommu, bus, devfn, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) if (!context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) if (context_present(context))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) * For kdump cases, old valid entries may be cached due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) * in-flight DMA and copied pgtable, but there is no unmapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) * behaviour for them, thus we need an explicit cache flush for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) * the newly-mapped device. For kdump, at this point, the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) * is supposed to finish reset at its driver probe stage, so no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) * in-flight DMA will exist, and we don't need to worry anymore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) * hereafter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) if (context_copied(context)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) u16 did_old = context_domain_id(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) if (did_old < cap_ndoms(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) iommu->flush.flush_context(iommu, did_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) (((u16)bus) << 8) | devfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) DMA_CCMD_MASK_NOBIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) DMA_CCMD_DEVICE_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) context_clear_entry(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) if (sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) unsigned long pds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) WARN_ON(!table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) /* Setup the PASID DIR pointer: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) pds = context_get_sm_pds(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) context->lo = (u64)virt_to_phys(table->table) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) context_pdts(pds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) /* Setup the RID_PASID field: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) context_set_sm_rid2pasid(context, PASID_RID2PASID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) * Setup the Device-TLB enable bit and Page request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) * Enable bit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) if (info && info->ats_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) context_set_sm_dte(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) if (info && info->pri_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) context_set_sm_pre(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) struct dma_pte *pgd = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) int agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) context_set_domain_id(context, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) if (translation != CONTEXT_TT_PASS_THROUGH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) * Skip top levels of page tables for iommu which has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) * less agaw than default. Unnecessary for PT mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) pgd = phys_to_virt(dma_pte_addr(pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) if (!dma_pte_present(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) if (info && info->ats_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) translation = CONTEXT_TT_DEV_IOTLB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) translation = CONTEXT_TT_MULTI_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) context_set_address_root(context, virt_to_phys(pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) context_set_address_width(context, agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) * In pass through mode, AW must be programmed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) * indicate the largest AGAW value supported by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) * hardware. And ASR is ignored by hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) context_set_address_width(context, iommu->msagaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) context_set_translation_type(context, translation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) context_set_fault_enable(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) context_set_present(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) if (!ecap_coherent(iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) clflush_cache_range(context, sizeof(*context));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) * It's a non-present to present mapping. If hardware doesn't cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) * non-present entry we only need to flush the write-buffer. If the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) * _does_ cache non-present entries, then it does so in the special
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) * domain #0, which we have to flush:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) if (cap_caching_mode(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) iommu->flush.flush_context(iommu, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) (((u16)bus) << 8) | devfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) DMA_CCMD_MASK_NOBIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) DMA_CCMD_DEVICE_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) iommu_enable_dev_iotlb(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) struct domain_context_mapping_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) struct pasid_table *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) static int domain_context_mapping_cb(struct pci_dev *pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) u16 alias, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) struct domain_context_mapping_data *data = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) return domain_context_mapping_one(data->domain, data->iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) data->table, PCI_BUS_NUM(alias),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) alias & 0xff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) domain_context_mapping(struct dmar_domain *domain, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) struct domain_context_mapping_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) struct pasid_table *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) u8 bus, devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) iommu = device_to_iommu(dev, &bus, &devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) table = intel_pasid_get_table(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) if (!dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) return domain_context_mapping_one(domain, iommu, table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) data.domain = domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) data.iommu = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) data.table = table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) return pci_for_each_dma_alias(to_pci_dev(dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) &domain_context_mapping_cb, &data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) static int domain_context_mapped_cb(struct pci_dev *pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) u16 alias, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) struct intel_iommu *iommu = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) static int domain_context_mapped(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) u8 bus, devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) iommu = device_to_iommu(dev, &bus, &devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) if (!dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) return device_context_mapped(iommu, bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) return !pci_for_each_dma_alias(to_pci_dev(dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) domain_context_mapped_cb, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) /* Returns a number of VTD pages, but aligned to MM page size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) static inline unsigned long aligned_nrpages(unsigned long host_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) host_addr &= ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) /* Return largest possible superpage level for a given mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) static inline int hardware_largepage_caps(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) unsigned long iov_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) unsigned long phy_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) unsigned long pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) int support, level = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) unsigned long pfnmerge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) support = domain->iommu_superpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) /* To use a large page, the virtual *and* physical addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) must be aligned to 2MiB/1GiB/etc. Lower bits set in either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) of them will mean we have to use smaller pages. So just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) merge them and check both at once. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) pfnmerge = iov_pfn | phy_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) pages >>= VTD_STRIDE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) if (!pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) pfnmerge >>= VTD_STRIDE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) level++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) support--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) return level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) struct scatterlist *sg, unsigned long phys_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) unsigned long nr_pages, int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) struct dma_pte *first_pte = NULL, *pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) phys_addr_t pteval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) unsigned long sg_res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) unsigned int largepage_lvl = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) unsigned long lvl_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) u64 attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) attr |= DMA_FL_PTE_PRESENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) if (domain_use_first_level(domain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) if (domain->domain.type == IOMMU_DOMAIN_DMA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) attr |= DMA_FL_PTE_ACCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) if (prot & DMA_PTE_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) attr |= DMA_FL_PTE_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) if (!sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) sg_res = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) while (nr_pages > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) uint64_t tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) if (!sg_res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) unsigned int pgoff = sg->offset & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) sg_res = aligned_nrpages(sg->offset, sg->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) sg->dma_length = sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) pteval = (sg_phys(sg) - pgoff) | attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) phys_pfn = pteval >> VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) if (!pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) /* It is large page*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) if (largepage_lvl > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) unsigned long nr_superpages, end_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) pteval |= DMA_PTE_LARGE_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) lvl_pages = lvl_to_nr_pages(largepage_lvl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) nr_superpages = sg_res / lvl_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) * Ensure that old small page tables are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) * removed to make room for superpage(s).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) * We're adding new large pages, so make sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) * we don't remove their parent tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) largepage_lvl + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) /* We don't need lock here, nobody else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) * touches the iova range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) if (tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) static int dumps = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) iov_pfn, tmp, (unsigned long long)pteval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) if (dumps) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) dumps--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) debug_dma_dump_mappings(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) lvl_pages = lvl_to_nr_pages(largepage_lvl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) BUG_ON(nr_pages < lvl_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) BUG_ON(sg_res < lvl_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) nr_pages -= lvl_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) iov_pfn += lvl_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) phys_pfn += lvl_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) pteval += lvl_pages * VTD_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) sg_res -= lvl_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) /* If the next PTE would be the first in a new page, then we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) need to flush the cache on the entries we've just written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) And then we'll need to recalculate 'pte', so clear it and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) let it get set again in the if (!pte) block above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) If we're done (!nr_pages) we need to flush the cache too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) Also if we've been setting superpages, we may need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) recalculate 'pte' and switch back to smaller pages for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) end of the mapping, if the trailing size is not enough to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) use another superpage (i.e. sg_res < lvl_pages). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) pte++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) if (!nr_pages || first_pte_in_page(pte) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) (largepage_lvl > 1 && sg_res < lvl_pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) domain_flush_cache(domain, first_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) (void *)pte - (void *)first_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) pte = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) if (!sg_res && nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) sg = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) struct scatterlist *sg, unsigned long phys_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) unsigned long nr_pages, int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) int iommu_id, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) /* Do the real mapping first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) for_each_domain_iommu(iommu_id, domain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) iommu = g_iommus[iommu_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) struct scatterlist *sg, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) unsigned long phys_pfn, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) u16 did_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) context = iommu_context_addr(iommu, bus, devfn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) if (!context) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) did_old = context_domain_id(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) context_clear_entry(context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) __iommu_flush_cache(iommu, context, sizeof(*context));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) iommu->flush.flush_context(iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) did_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) (((u16)bus) << 8) | devfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) DMA_CCMD_MASK_NOBIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) DMA_CCMD_DEVICE_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) if (sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) iommu->flush.flush_iotlb(iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) did_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) static inline void unlink_domain_info(struct device_domain_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) list_del(&info->link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) list_del(&info->global);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) if (info->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) dev_iommu_priv_set(info->dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) static void domain_remove_dev_info(struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) struct device_domain_info *info, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) list_for_each_entry_safe(info, tmp, &domain->devices, link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) __dmar_remove_one_dev_info(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) struct dmar_domain *find_domain(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) if (unlikely(!dev || !dev->iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) if (unlikely(attach_deferred(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) /* No lock here, assumes no domain exit in normal case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) if (likely(info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) return info->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) static void do_deferred_attach(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) struct iommu_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) dev_iommu_priv_set(dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) domain = iommu_get_domain_for_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) if (domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) intel_iommu_attach_device(domain, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) static inline struct device_domain_info *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) list_for_each_entry(info, &device_domain_list, global)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) if (info->segment == segment && info->bus == bus &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) info->devfn == devfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) return info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) static int domain_setup_first_level(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) struct dma_pte *pgd = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) int agaw, level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) int flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) * Skip top levels of page tables for iommu which has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) * less agaw than default. Unnecessary for PT mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) pgd = phys_to_virt(dma_pte_addr(pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) if (!dma_pte_present(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) level = agaw_to_level(agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) if (level != 4 && level != 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) if (pasid != PASID_RID2PASID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) flags |= PASID_FLAG_SUPERVISOR_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) if (level == 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) flags |= PASID_FLAG_FL5LP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) flags |= PASID_FLAG_PAGE_SNOOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) domain->iommu_did[iommu->seq_id],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) static bool dev_is_real_dma_subdevice(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) return dev && dev_is_pci(dev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) int bus, int devfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) struct dmar_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) struct dmar_domain *found = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) info = alloc_devinfo_mem();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) if (!dev_is_real_dma_subdevice(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) info->bus = bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) info->devfn = devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) info->segment = iommu->segment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) info->bus = pdev->bus->number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) info->devfn = pdev->devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) info->segment = pci_domain_nr(pdev->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) info->ats_supported = info->pasid_supported = info->pri_supported = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) info->ats_qdep = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) info->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) info->domain = domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) info->iommu = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) info->pasid_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) info->auxd_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) INIT_LIST_HEAD(&info->auxiliary_domains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) if (dev && dev_is_pci(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) struct pci_dev *pdev = to_pci_dev(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) if (ecap_dev_iotlb_support(iommu->ecap) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) pci_ats_supported(pdev) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) dmar_find_matched_atsr_unit(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) info->ats_supported = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) if (sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) if (pasid_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) int features = pci_pasid_features(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) if (features >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) info->pasid_supported = features | 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) if (info->ats_supported && ecap_prs(iommu->ecap) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) pci_pri_supported(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) info->pri_supported = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) if (dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) found = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) if (!found) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) struct device_domain_info *info2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) info2 = dmar_search_domain_by_dev_info(info->segment, info->bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) info->devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) if (info2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) found = info2->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) info2->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) if (found) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) free_devinfo_mem(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) /* Caller must free the original domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) ret = domain_attach_iommu(domain, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) free_devinfo_mem(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) list_add(&info->link, &domain->devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) list_add(&info->global, &device_domain_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) if (dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) dev_iommu_priv_set(dev, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) /* PASID table is mandatory for a PCI device in scalable mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) ret = intel_pasid_alloc_table(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) dev_err(dev, "PASID table allocation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) /* Setup the PASID entry for requests without PASID: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) if (hw_pass_through && domain_type_is_si(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) ret = intel_pasid_setup_pass_through(iommu, domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) dev, PASID_RID2PASID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) else if (domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) ret = domain_setup_first_level(iommu, domain, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) PASID_RID2PASID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) ret = intel_pasid_setup_second_level(iommu, domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) dev, PASID_RID2PASID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) dev_err(dev, "Setup RID2PASID failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) if (dev && domain_context_mapping(domain, dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) dev_err(dev, "Domain context map failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) return domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) static int iommu_domain_identity_map(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) unsigned long first_vpfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) unsigned long last_vpfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) * RMRR range might have overlap with physical memory range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) * clear it first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) dma_pte_clear_range(domain, first_vpfn, last_vpfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) return __domain_mapping(domain, first_vpfn, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) first_vpfn, last_vpfn - first_vpfn + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) DMA_PTE_READ|DMA_PTE_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) static int md_domain_init(struct dmar_domain *domain, int guest_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) static int __init si_domain_init(int hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) struct dmar_rmrr_unit *rmrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) int i, nid, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) if (!si_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) domain_exit(si_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) if (hw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) for_each_online_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) unsigned long start_pfn, end_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) ret = iommu_domain_identity_map(si_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) mm_to_dma_pfn(start_pfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) mm_to_dma_pfn(end_pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) * Identity map the RMRRs so that devices with RMRRs could also use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) * the si_domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) for_each_rmrr_units(rmrr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) i, dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) unsigned long long start = rmrr->base_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) unsigned long long end = rmrr->end_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) if (WARN_ON(end < start ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) end >> agaw_to_width(si_domain->agaw)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) ret = iommu_domain_identity_map(si_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) mm_to_dma_pfn(start >> PAGE_SHIFT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) mm_to_dma_pfn(end >> PAGE_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) struct dmar_domain *ndomain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) u8 bus, devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) iommu = device_to_iommu(dev, &bus, &devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) if (ndomain != domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) static bool device_has_rmrr(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) struct dmar_rmrr_unit *rmrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) struct device *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) for_each_rmrr_units(rmrr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) * Return TRUE if this RMRR contains the device that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) * is passed in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) for_each_active_dev_scope(rmrr->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) rmrr->devices_cnt, i, tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) if (tmp == dev ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) is_downstream_to_pci_bridge(dev, tmp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * device_rmrr_is_relaxable - Test whether the RMRR of this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * is relaxable (ie. is allowed to be not enforced under some conditions)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * @dev: device handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) * We assume that PCI USB devices with RMRRs have them largely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) * for historical reasons and that the RMRR space is not actively used post
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) * boot. This exclusion may change if vendors begin to abuse it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) * The same exception is made for graphics devices, with the requirement that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) * any use of the RMRR regions will be torn down before assigning the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) * to a guest.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) * Return: true if the RMRR is relaxable, false otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) static bool device_rmrr_is_relaxable(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) if (!dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) * There are a couple cases where we need to restrict the functionality of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) * devices associated with RMRRs. The first is when evaluating a device for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) * identity mapping because problems exist when devices are moved in and out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) * of domains and their respective RMRR information is lost. This means that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) * a device with associated RMRRs will never be in a "passthrough" domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) * The second is use of the device through the IOMMU API. This interface
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) * expects to have full control of the IOVA space for the device. We cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) * satisfy both the requirement that RMRR access is maintained and have an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) * unencumbered IOVA space. We also have no ability to quiesce the device's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) * use of the RMRR space or even inform the IOMMU API user of the restriction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) * We therefore prevent devices associated with an RMRR from participating in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) * the IOMMU API, which eliminates them from device assignment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) * In both cases, devices which have relaxable RMRRs are not concerned by this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) * restriction. See device_rmrr_is_relaxable comment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) static bool device_is_rmrr_locked(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) if (!device_has_rmrr(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) if (device_rmrr_is_relaxable(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) * Return the required default domain type for a specific device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) * @dev: the device in query
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) * @startup: true if this is during early boot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) * - 0: both identity and dynamic domains work for this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) static int device_def_domain_type(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) if (dev_is_pci(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) * Prevent any device marked as untrusted from getting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) * placed into the statically identity mapping domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) if (pdev->untrusted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) return IOMMU_DOMAIN_DMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) return IOMMU_DOMAIN_IDENTITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) return IOMMU_DOMAIN_IDENTITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) static void intel_iommu_init_qi(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) * Start from the sane iommu hardware state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) * If the queued invalidation is already initialized by us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) * (for example, while enabling interrupt-remapping) then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) * we got the things already rolling from a sane state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) if (!iommu->qi) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) * Clear any previous faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) dmar_fault(-1, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) * Disable queued invalidation if supported and already enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) * before OS handover.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) dmar_disable_qi(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) if (dmar_enable_qi(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) * Queued Invalidate not enabled, use Register Based Invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) iommu->flush.flush_context = __iommu_flush_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) iommu->flush.flush_iotlb = __iommu_flush_iotlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) pr_info("%s: Using Register based invalidation\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) iommu->flush.flush_context = qi_flush_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) iommu->flush.flush_iotlb = qi_flush_iotlb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) pr_info("%s: Using Queued invalidation\n", iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) static int copy_context_table(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) struct root_entry *old_re,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) struct context_entry **tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) int bus, bool ext)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) struct context_entry *new_ce = NULL, ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) struct context_entry *old_ce = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) struct root_entry re;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) phys_addr_t old_ce_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) tbl_idx = ext ? bus * 2 : bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) memcpy(&re, old_re, sizeof(re));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) for (devfn = 0; devfn < 256; devfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) /* First calculate the correct index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) idx = (ext ? devfn * 2 : devfn) % 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) if (idx == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) /* First save what we may have and clean up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) if (new_ce) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) tbl[tbl_idx] = new_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) __iommu_flush_cache(iommu, new_ce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) VTD_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) pos = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) if (old_ce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) memunmap(old_ce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) if (devfn < 0x80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) old_ce_phys = root_entry_lctp(&re);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) old_ce_phys = root_entry_uctp(&re);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (!old_ce_phys) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) if (ext && devfn == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) /* No LCTP, try UCTP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) devfn = 0x7f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) old_ce = memremap(old_ce_phys, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) MEMREMAP_WB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) if (!old_ce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) new_ce = alloc_pgtable_page(iommu->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) if (!new_ce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) goto out_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) /* Now copy the context entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) memcpy(&ce, old_ce + idx, sizeof(ce));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) if (!__context_present(&ce))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) did = context_domain_id(&ce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) if (did >= 0 && did < cap_ndoms(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) set_bit(did, iommu->domain_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) * We need a marker for copied context entries. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) * marker needs to work for the old format as well as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) * for extended context entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) * Bit 67 of the context entry is used. In the old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) * format this bit is available to software, in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) * extended format it is the PGE bit, but PGE is ignored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) * by HW if PASIDs are disabled (and thus still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) * available).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) * So disable PASIDs first and then mark the entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) * copied. This means that we don't copy PASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) * translations from the old kernel, but this is fine as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) * faults there are not fatal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) context_clear_pasid_enable(&ce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) context_set_copied(&ce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) new_ce[idx] = ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) tbl[tbl_idx + pos] = new_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) out_unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) memunmap(old_ce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) static int copy_translation_tables(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) struct context_entry **ctxt_tbls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) struct root_entry *old_rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) phys_addr_t old_rt_phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) int ctxt_table_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) u64 rtaddr_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) int bus, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) bool new_ext, ext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) new_ext = !!ecap_ecs(iommu->ecap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) * The RTT bit can only be changed when translation is disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) * but disabling translation means to open a window for data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) * corruption. So bail out and don't copy anything if we would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) * have to change the bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) if (new_ext != ext)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) if (!old_rt_phys)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) if (!old_rt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) /* This is too big for the stack - allocate it from slab */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) ctxt_table_entries = ext ? 512 : 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) if (!ctxt_tbls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) goto out_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) for (bus = 0; bus < 256; bus++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) ret = copy_context_table(iommu, &old_rt[bus],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) ctxt_tbls, bus, ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) pr_err("%s: Failed to copy context table for bus %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) iommu->name, bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) /* Context tables are copied, now write them to the root_entry table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) for (bus = 0; bus < 256; bus++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) int idx = ext ? bus * 2 : bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) if (ctxt_tbls[idx]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) val = virt_to_phys(ctxt_tbls[idx]) | 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) iommu->root_entry[bus].lo = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) if (!ext || !ctxt_tbls[idx + 1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) iommu->root_entry[bus].hi = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) kfree(ctxt_tbls);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) out_unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) memunmap(old_rt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) static ioasid_t intel_vcmd_ioasid_alloc(ioasid_t min, ioasid_t max, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) struct intel_iommu *iommu = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) ioasid_t ioasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * VT-d virtual command interface always uses the full 20 bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * PASID range. Host can partition guest PASID range based on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) * policies but it is out of guest's control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) if (min < PASID_MIN || max > intel_pasid_max_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) if (vcmd_alloc_pasid(iommu, &ioasid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) return INVALID_IOASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) return ioasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) static void intel_vcmd_ioasid_free(ioasid_t ioasid, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) struct intel_iommu *iommu = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) * Sanity check the ioasid owner is done at upper layer, e.g. VFIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) * We can only free the PASID when all the devices are unbound.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) if (ioasid_find(NULL, ioasid, NULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) pr_alert("Cannot free active IOASID %d\n", ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) vcmd_free_pasid(iommu, ioasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) static void register_pasid_allocator(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) * If we are running in the host, no need for custom allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) * in that PASIDs are allocated from the host system-wide.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) if (!cap_caching_mode(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) if (!sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) pr_warn("VT-d Scalable Mode not enabled, no PASID allocation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) * Register a custom PASID allocator if we are running in a guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) * guest PASID must be obtained via virtual command interface.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) * There can be multiple vIOMMUs in each guest but only one allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) * is active. All vIOMMU allocators will eventually be calling the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) * host allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) if (!vccap_pasid(iommu->vccap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) pr_info("Register custom PASID allocator\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) iommu->pasid_allocator.alloc = intel_vcmd_ioasid_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) iommu->pasid_allocator.free = intel_vcmd_ioasid_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) iommu->pasid_allocator.pdata = (void *)iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) if (ioasid_register_allocator(&iommu->pasid_allocator)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) pr_warn("Custom PASID allocator failed, scalable mode disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) * Disable scalable mode on this IOMMU if there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) * is no custom allocator. Mixing SM capable vIOMMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) * and non-SM vIOMMU are not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) intel_iommu_sm = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) static int __init init_dmars(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) * for each drhd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) * allocate root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) * initialize and program root entry to not present
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) * endfor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) for_each_drhd_unit(drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) * lock not needed as this is only incremented in the single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) * threaded kernel __init code path all other access are read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) * only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) g_num_of_iommus++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) /* Preallocate enough resources for IOMMU hot-addition */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) g_num_of_iommus = DMAR_UNITS_SUPPORTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) if (!g_iommus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) pr_err("Allocating global iommu array failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) for_each_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) if (drhd->ignored) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) * Find the max pasid size of all IOMMU's in the system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) * We need to ensure the system pasid table is no bigger
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) * than the smallest supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) if (pasid_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) u32 temp = 2 << ecap_pss(iommu->ecap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) intel_pasid_max_id = min_t(u32, temp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) intel_pasid_max_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) g_iommus[iommu->seq_id] = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) intel_iommu_init_qi(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) ret = iommu_init_domains(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) goto free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) init_translation_status(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) clear_translation_pre_enabled(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) * TBD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) * we could share the same root & context tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) * among all IOMMU's. Need to Split it later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) ret = iommu_alloc_root_entry(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) goto free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) if (translation_pre_enabled(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) pr_info("Translation already enabled - trying to copy translation structures\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) ret = copy_translation_tables(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) * We found the IOMMU with translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) * enabled - but failed to copy over the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) * old root-entry table. Try to proceed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) * by disabling translation now and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) * allocating a clean root-entry table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) * This might cause DMAR faults, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) * probably the dump will still succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) pr_err("Failed to copy translation tables from previous kernel for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) clear_translation_pre_enabled(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) pr_info("Copied translation tables from previous kernel for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) if (!ecap_pass_through(iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) hw_pass_through = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) if (!intel_iommu_strict && cap_caching_mode(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) pr_warn("Disable batched IOTLB flush due to virtualization");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) intel_iommu_strict = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) intel_svm_check(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) * Now that qi is enabled on all iommus, set the root entry and flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) * caches. This is required on some Intel X58 chipsets, otherwise the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) * flush_context function will loop forever and the boot hangs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) register_pasid_allocator(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) iommu_set_root_entry(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) dmar_map_gfx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) if (!dmar_map_gfx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) iommu_identity_mapping |= IDENTMAP_GFX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) check_tylersburg_isoch();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) ret = si_domain_init(hw_pass_through);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) goto free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) * for each drhd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) * enable fault log
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) * global invalidate context cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) * global invalidate iotlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) * enable translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) for_each_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) if (drhd->ignored) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) * we always have to disable PMRs or DMA may fail on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) * this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) * Call dmar_alloc_hwirq() with dmar_global_lock held,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) * could cause possible lock race condition.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) up_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) ret = intel_svm_enable_prq(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) down_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) goto free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) ret = dmar_set_interrupt(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) goto free_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) free_iommu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) disable_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) free_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) kfree(g_iommus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) /* This takes a number of _MM_ pages, not VTD pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) static unsigned long intel_alloc_iova(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) unsigned long nrpages, uint64_t dma_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) unsigned long iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) * Restrict dma_mask to the width that the iommu can handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) * First-level translation restricts the input-address to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) * canonical address (i.e., address bits 63:N have the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) * value as address bit [N-1], where N is 48-bits with 4-level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) * paging and 57-bits with 5-level paging). Hence, skip bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) * [N-1].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) if (domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) /* Ensure we reserve the whole size-aligned region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) nrpages = __roundup_pow_of_two(nrpages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) * First try to allocate an io virtual address in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) * DMA_BIT_MASK(32) and if that fails then try allocating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) * from higher range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) IOVA_PFN(DMA_BIT_MASK(32)), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) if (iova_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) return iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) IOVA_PFN(dma_mask), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) if (unlikely(!iova_pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) dev_err_once(dev, "Allocating %ld-page iova failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) nrpages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) return iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) size_t size, int dir, u64 dma_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) phys_addr_t start_paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) unsigned long iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) int prot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) if (unlikely(attach_deferred(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) do_deferred_attach(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) if (!domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) iommu = domain_get_iommu(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) size = aligned_nrpages(paddr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) if (!iova_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) * Check if DMAR supports zero-length reads on write only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) * mappings..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) !cap_zlr(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) prot |= DMA_PTE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) prot |= DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) * paddr - (paddr + size) might be partial page, we should map the whole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) * page. Note: if two part of one page are separately mapped, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) * might have two guest_addr mapping to the same host paddr, but this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) * is not a big problem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) mm_to_dma_pfn(paddr_pfn), size, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) start_paddr += paddr & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) trace_map_single(dev, start_paddr, paddr, size << VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) return start_paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) if (iova_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) size, (unsigned long long)paddr, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) static dma_addr_t intel_map_page(struct device *dev, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) unsigned long offset, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) enum dma_data_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) return __intel_map_single(dev, page_to_phys(page) + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) size, dir, *dev->dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) size_t size, enum dma_data_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) unsigned long start_pfn, last_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) unsigned long nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) unsigned long iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) struct page *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) struct pci_dev *pdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) BUG_ON(!domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) iommu = domain_get_iommu(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) iova_pfn = IOVA_PFN(dev_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) nrpages = aligned_nrpages(dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) start_pfn = mm_to_dma_pfn(iova_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) last_pfn = start_pfn + nrpages - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) if (dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) freelist = domain_unmap(domain, start_pfn, last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) if (intel_iommu_strict || (pdev && pdev->untrusted) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) !has_iova_flush_queue(&domain->iovad)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) iommu_flush_iotlb_psi(iommu, domain, start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) nrpages, !freelist, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) /* free iova */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) dma_free_pagelist(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) queue_iova(&domain->iovad, iova_pfn, nrpages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) (unsigned long)freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) * queue up the release of the unmap to save the 1/6th of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) * cpu used up by the iotlb flush operation...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) trace_unmap_single(dev, dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) size_t size, enum dma_data_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) intel_unmap(dev, dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) size_t size, enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) intel_unmap(dev, dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) static void *intel_alloc_coherent(struct device *dev, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) dma_addr_t *dma_handle, gfp_t flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) struct page *page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) if (unlikely(attach_deferred(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) do_deferred_attach(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) size = PAGE_ALIGN(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) if (gfpflags_allow_blocking(flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) unsigned int count = size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) page = dma_alloc_from_contiguous(dev, count, order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) flags & __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) page = alloc_pages(flags, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) memset(page_address(page), 0, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) DMA_BIDIRECTIONAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) dev->coherent_dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) if (*dma_handle != DMA_MAPPING_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) return page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) __free_pages(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) dma_addr_t dma_handle, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) struct page *page = virt_to_page(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) size = PAGE_ALIGN(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) intel_unmap(dev, dma_handle, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) __free_pages(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) int nelems, enum dma_data_direction dir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) unsigned long nrpages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) for_each_sg(sglist, sg, nelems, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) trace_unmap_sg(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) size_t size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) int prot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) unsigned long iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) unsigned long start_vpfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) BUG_ON(dir == DMA_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) if (unlikely(attach_deferred(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) do_deferred_attach(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) if (!domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) iommu = domain_get_iommu(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) size += aligned_nrpages(sg->offset, sg->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) *dev->dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) if (!iova_pfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) sglist->dma_length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) * Check if DMAR supports zero-length reads on write only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) * mappings..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) !cap_zlr(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) prot |= DMA_PTE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) prot |= DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) start_vpfn = mm_to_dma_pfn(iova_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) dma_pte_free_pagetable(domain, start_vpfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) start_vpfn + size - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) agaw_to_level(domain->agaw) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) trace_map_sg(dev, i + 1, nelems, sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) return nelems;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) static u64 intel_get_required_mask(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) return DMA_BIT_MASK(32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) static const struct dma_map_ops intel_dma_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) .alloc = intel_alloc_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) .free = intel_free_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) .map_sg = intel_map_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) .unmap_sg = intel_unmap_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) .map_page = intel_map_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) .unmap_page = intel_unmap_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) .map_resource = intel_map_resource,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) .unmap_resource = intel_unmap_resource,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) .dma_supported = dma_direct_supported,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) .mmap = dma_common_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) .get_sgtable = dma_common_get_sgtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) .alloc_pages = dma_common_alloc_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) .free_pages = dma_common_free_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) .get_required_mask = intel_get_required_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) bounce_sync_single(struct device *dev, dma_addr_t addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) enum dma_data_direction dir, enum dma_sync_target target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) phys_addr_t tlb_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) if (WARN_ON(!domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) tlb_addr = intel_iommu_iova_to_phys(&domain->domain, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) if (is_swiotlb_buffer(tlb_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) swiotlb_tbl_sync_single(dev, tlb_addr, size, dir, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) static dma_addr_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) enum dma_data_direction dir, unsigned long attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) u64 dma_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) unsigned long iova_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) unsigned long nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) phys_addr_t tlb_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) int prot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) if (unlikely(attach_deferred(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) do_deferred_attach(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) if (WARN_ON(dir == DMA_NONE || !domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) iommu = domain_get_iommu(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) if (WARN_ON(!iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) nrpages = aligned_nrpages(0, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) iova_pfn = intel_alloc_iova(dev, domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) dma_to_mm_pfn(nrpages), dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) if (!iova_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) * Check if DMAR supports zero-length reads on write only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) * mappings..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) !cap_zlr(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) prot |= DMA_PTE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) prot |= DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) * If both the physical buffer start address and size are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) * page aligned, we don't need to use a bounce page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) tlb_addr = swiotlb_tbl_map_single(dev, paddr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) aligned_size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) if (tlb_addr == DMA_MAPPING_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) goto swiotlb_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) /* Cleanup the padding area. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) void *padding_start = phys_to_virt(tlb_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) size_t padding_size = aligned_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) (dir == DMA_TO_DEVICE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) dir == DMA_BIDIRECTIONAL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) padding_start += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) padding_size -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) memset(padding_start, 0, padding_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) tlb_addr = paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) tlb_addr >> VTD_PAGE_SHIFT, nrpages, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) goto mapping_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) trace_bounce_map_single(dev, iova_pfn << PAGE_SHIFT, paddr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) return (phys_addr_t)iova_pfn << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) mapping_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) if (is_swiotlb_buffer(tlb_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) swiotlb_tbl_unmap_single(dev, tlb_addr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) aligned_size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) swiotlb_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) dev_err(dev, "Device bounce map: %zx@%llx dir %d --- failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) size, (unsigned long long)paddr, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) return DMA_MAPPING_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) bounce_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) phys_addr_t tlb_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) if (WARN_ON(!domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) tlb_addr = intel_iommu_iova_to_phys(&domain->domain, dev_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) if (WARN_ON(!tlb_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) intel_unmap(dev, dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) if (is_swiotlb_buffer(tlb_addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) swiotlb_tbl_unmap_single(dev, tlb_addr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) aligned_size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) trace_bounce_unmap_single(dev, dev_addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) static dma_addr_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) bounce_map_page(struct device *dev, struct page *page, unsigned long offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) size_t size, enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) return bounce_map_single(dev, page_to_phys(page) + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) size, dir, attrs, *dev->dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) static dma_addr_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) bounce_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) return bounce_map_single(dev, phys_addr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) dir, attrs, *dev->dma_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) bounce_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) bounce_unmap_single(dev, dev_addr, size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) bounce_unmap_resource(struct device *dev, dma_addr_t dev_addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) bounce_unmap_single(dev, dev_addr, size, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) bounce_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) bounce_unmap_page(dev, sg->dma_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) sg_dma_len(sg), dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) for_each_sg(sglist, sg, nelems, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) sg->dma_address = bounce_map_page(dev, sg_page(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) sg->offset, sg->length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) if (sg->dma_address == DMA_MAPPING_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) goto out_unmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) sg_dma_len(sg) = sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) trace_bounce_map_sg(dev, i + 1, nelems, sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) return nelems;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) out_unmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) bounce_unmap_sg(dev, sglist, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) bounce_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) size_t size, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) bounce_sync_single(dev, addr, size, dir, SYNC_FOR_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) bounce_sync_single_for_device(struct device *dev, dma_addr_t addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) size_t size, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) bounce_sync_single(dev, addr, size, dir, SYNC_FOR_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) bounce_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) int nelems, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) bounce_sync_single(dev, sg_dma_address(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) sg_dma_len(sg), dir, SYNC_FOR_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) bounce_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) int nelems, enum dma_data_direction dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) for_each_sg(sglist, sg, nelems, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) bounce_sync_single(dev, sg_dma_address(sg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) sg_dma_len(sg), dir, SYNC_FOR_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) static const struct dma_map_ops bounce_dma_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) .alloc = intel_alloc_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) .free = intel_free_coherent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) .map_sg = bounce_map_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) .unmap_sg = bounce_unmap_sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) .map_page = bounce_map_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) .unmap_page = bounce_unmap_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) .sync_single_for_cpu = bounce_sync_single_for_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) .sync_single_for_device = bounce_sync_single_for_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) .sync_sg_for_cpu = bounce_sync_sg_for_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) .sync_sg_for_device = bounce_sync_sg_for_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) .map_resource = bounce_map_resource,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) .unmap_resource = bounce_unmap_resource,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) .alloc_pages = dma_common_alloc_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) .free_pages = dma_common_free_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) .dma_supported = dma_direct_supported,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) static inline int iommu_domain_cache_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) iommu_domain_cache = kmem_cache_create("iommu_domain",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) sizeof(struct dmar_domain),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) SLAB_HWCACHE_ALIGN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) if (!iommu_domain_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) pr_err("Couldn't create iommu_domain cache\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) static inline int iommu_devinfo_cache_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) sizeof(struct device_domain_info),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) SLAB_HWCACHE_ALIGN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) if (!iommu_devinfo_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) pr_err("Couldn't create devinfo cache\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) static int __init iommu_init_mempool(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) ret = iova_cache_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) ret = iommu_domain_cache_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) goto domain_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) ret = iommu_devinfo_cache_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) kmem_cache_destroy(iommu_domain_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) domain_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) iova_cache_put();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) static void __init iommu_exit_mempool(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) kmem_cache_destroy(iommu_devinfo_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) kmem_cache_destroy(iommu_domain_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) iova_cache_put();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) static void __init init_no_remapping_devices(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) for_each_drhd_unit(drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) if (!drhd->include_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) for_each_active_dev_scope(drhd->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) drhd->devices_cnt, i, dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) /* ignore DMAR unit if no devices exist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) if (i == drhd->devices_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) drhd->ignored = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) for_each_active_drhd_unit(drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) if (drhd->include_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) for_each_active_dev_scope(drhd->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) drhd->devices_cnt, i, dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) if (i < drhd->devices_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) /* This IOMMU has *only* gfx devices. Either bypass it or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) set the gfx_mapped flag, as appropriate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) drhd->gfx_dedicated = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) if (!dmar_map_gfx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) drhd->ignored = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) #ifdef CONFIG_SUSPEND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) static int init_iommu_hw(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) struct intel_iommu *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) for_each_active_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) if (iommu->qi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) dmar_reenable_qi(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) for_each_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) if (drhd->ignored) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) * we always have to disable PMRs or DMA may fail on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) * this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) iommu_set_root_entry(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) iommu_enable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) static void iommu_flush_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) iommu->flush.flush_context(iommu, 0, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) DMA_CCMD_GLOBAL_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) iommu->flush.flush_iotlb(iommu, 0, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) DMA_TLB_GLOBAL_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) static int iommu_suspend(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) struct intel_iommu *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) if (!iommu->iommu_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) iommu_flush_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) iommu->iommu_state[SR_DMAR_FECTL_REG] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) readl(iommu->reg + DMAR_FECTL_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) iommu->iommu_state[SR_DMAR_FEDATA_REG] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) readl(iommu->reg + DMAR_FEDATA_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) iommu->iommu_state[SR_DMAR_FEADDR_REG] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) readl(iommu->reg + DMAR_FEADDR_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) readl(iommu->reg + DMAR_FEUADDR_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) for_each_active_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) kfree(iommu->iommu_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) static void iommu_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) struct intel_iommu *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) unsigned long flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) if (init_iommu_hw()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) raw_spin_lock_irqsave(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) iommu->reg + DMAR_FECTL_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) iommu->reg + DMAR_FEDATA_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) iommu->reg + DMAR_FEADDR_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) iommu->reg + DMAR_FEUADDR_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) for_each_active_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) kfree(iommu->iommu_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) static struct syscore_ops iommu_syscore_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) .resume = iommu_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) .suspend = iommu_suspend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) static void __init init_iommu_pm_ops(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) register_syscore_ops(&iommu_syscore_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) static inline void init_iommu_pm_ops(void) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) #endif /* CONFIG_PM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) static int rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) rmrr->end_address <= rmrr->base_address ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) arch_rmrr_sanity_check(rmrr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) struct acpi_dmar_reserved_memory *rmrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) struct dmar_rmrr_unit *rmrru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) rmrr = (struct acpi_dmar_reserved_memory *)header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) if (rmrr_sanity_check(rmrr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) pr_warn(FW_BUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) rmrr->base_address, rmrr->end_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) dmi_get_system_info(DMI_BIOS_VENDOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) dmi_get_system_info(DMI_BIOS_VERSION),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) dmi_get_system_info(DMI_PRODUCT_VERSION));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) if (!rmrru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) rmrru->hdr = header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) rmrru->base_address = rmrr->base_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) rmrru->end_address = rmrr->end_address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) ((void *)rmrr) + rmrr->header.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) &rmrru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) if (rmrru->devices_cnt && rmrru->devices == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) goto free_rmrru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) list_add(&rmrru->list, &dmar_rmrr_units);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) free_rmrru:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) kfree(rmrru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) struct acpi_dmar_atsr *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) dmar_rcu_check()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) tmp = (struct acpi_dmar_atsr *)atsru->hdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) if (atsr->segment != tmp->segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) if (atsr->header.length != tmp->header.length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) if (memcmp(atsr, tmp, atsr->header.length) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) return atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) struct acpi_dmar_atsr *atsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) atsr = container_of(hdr, struct acpi_dmar_atsr, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) atsru = dmar_find_atsr(atsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) if (atsru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) if (!atsru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) * If memory is allocated from slab by ACPI _DSM method, we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) * copy the memory content because the memory buffer will be freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) * on return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) atsru->hdr = (void *)(atsru + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) memcpy(atsru->hdr, hdr, hdr->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) atsru->include_all = atsr->flags & 0x1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) if (!atsru->include_all) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) (void *)atsr + atsr->header.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) &atsru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) if (atsru->devices_cnt && atsru->devices == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) kfree(atsru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) list_add_rcu(&atsru->list, &dmar_atsr_units);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) kfree(atsru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) struct acpi_dmar_atsr *atsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) atsr = container_of(hdr, struct acpi_dmar_atsr, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) atsru = dmar_find_atsr(atsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) if (atsru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) list_del_rcu(&atsru->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) intel_iommu_free_atsr(atsru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) struct acpi_dmar_atsr *atsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) atsr = container_of(hdr, struct acpi_dmar_atsr, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) atsru = dmar_find_atsr(atsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) if (!atsru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) i, dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) int sp, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) struct intel_iommu *iommu = dmaru->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) if (g_iommus[iommu->seq_id])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) pr_warn("%s: Doesn't support hardware pass through.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) if (!ecap_sc_support(iommu->ecap) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) domain_update_iommu_snooping(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) pr_warn("%s: Doesn't support snooping.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) sp = domain_update_iommu_superpage(NULL, iommu) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) pr_warn("%s: Doesn't support large page.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) * Disable translation if already enabled prior to OS handover.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) if (iommu->gcmd & DMA_GCMD_TE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) g_iommus[iommu->seq_id] = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) ret = iommu_init_domains(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) ret = iommu_alloc_root_entry(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) intel_svm_check(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) if (dmaru->ignored) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) * we always have to disable PMRs or DMA may fail on this device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) intel_iommu_init_qi(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) ret = intel_svm_enable_prq(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) goto disable_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) ret = dmar_set_interrupt(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) goto disable_iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) iommu_set_root_entry(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) iommu_enable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) disable_iommu:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) disable_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) free_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) struct intel_iommu *iommu = dmaru->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) if (!intel_iommu_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) if (iommu == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) if (insert) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) ret = intel_iommu_add(dmaru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) disable_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) free_dmar_iommu(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) static void intel_iommu_free_dmars(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) struct dmar_rmrr_unit *rmrru, *rmrr_n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) struct dmar_atsr_unit *atsru, *atsr_n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) list_del(&rmrru->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) kfree(rmrru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) list_del(&atsru->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) intel_iommu_free_atsr(atsru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) int dmar_find_matched_atsr_unit(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) int i, ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) struct pci_bus *bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) struct pci_dev *bridge = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) struct device *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) struct acpi_dmar_atsr *atsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) dev = pci_physfn(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) for (bus = dev->bus; bus; bus = bus->parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) bridge = bus->self;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) /* If it's an integrated device, allow ATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) if (!bridge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) /* Connected via non-PCIe: no ATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) if (!pci_is_pcie(bridge) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) /* If we found the root port, look it up in the ATSR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) if (atsr->segment != pci_domain_nr(dev->bus))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) if (tmp == &bridge->dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) if (atsru->include_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) struct dmar_rmrr_unit *rmrru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) struct dmar_atsr_unit *atsru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) struct acpi_dmar_atsr *atsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) struct acpi_dmar_reserved_memory *rmrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) rmrr = container_of(rmrru->hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) struct acpi_dmar_reserved_memory, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) if (info->event == BUS_NOTIFY_ADD_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) ((void *)rmrr) + rmrr->header.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) rmrr->segment, rmrru->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) rmrru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) dmar_remove_dev_scope(info, rmrr->segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) rmrru->devices, rmrru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) list_for_each_entry(atsru, &dmar_atsr_units, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) if (atsru->include_all)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) if (info->event == BUS_NOTIFY_ADD_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) (void *)atsr + atsr->header.length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) atsr->segment, atsru->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) atsru->devices_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) else if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) if (dmar_remove_dev_scope(info, atsr->segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) atsru->devices, atsru->devices_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) static int intel_iommu_memory_notifier(struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) unsigned long val, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) struct memory_notify *mhp = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) unsigned long start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) unsigned long last_vpfn = mm_to_dma_pfn(mhp->start_pfn +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) mhp->nr_pages - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) switch (val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) case MEM_GOING_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) if (iommu_domain_identity_map(si_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) start_vpfn, last_vpfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) pr_warn("Failed to build identity map for [%lx-%lx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) start_vpfn, last_vpfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) return NOTIFY_BAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) case MEM_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) case MEM_CANCEL_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) struct page *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) freelist = domain_unmap(si_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) start_vpfn, last_vpfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) for_each_active_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) iommu_flush_iotlb_psi(iommu, si_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) start_vpfn, mhp->nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) !freelist, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) dma_free_pagelist(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) static struct notifier_block intel_iommu_memory_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) .notifier_call = intel_iommu_memory_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) .priority = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) static void free_all_cpu_cached_iovas(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) for (i = 0; i < g_num_of_iommus; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) struct intel_iommu *iommu = g_iommus[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) int did;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) for (did = 0; did < cap_ndoms(iommu->cap); did++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) domain = get_iommu_domain(iommu, (u16)did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) free_cpu_cached_iovas(cpu, &domain->iovad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) static int intel_iommu_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) free_all_cpu_cached_iovas(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) static void intel_disable_iommus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) struct intel_iommu *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) for_each_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) iommu_disable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) void intel_iommu_shutdown(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) struct intel_iommu *iommu = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) if (no_iommu || dmar_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) down_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) /* Disable PMRs explicitly here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) for_each_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) /* Make sure the IOMMUs are switched off */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) intel_disable_iommus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) up_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) return container_of(iommu_dev, struct intel_iommu, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) static ssize_t intel_iommu_show_version(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) u32 ver = readl(iommu->reg + DMAR_VER_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) return sprintf(buf, "%d:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) static ssize_t intel_iommu_show_address(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) return sprintf(buf, "%llx\n", iommu->reg_phys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) static ssize_t intel_iommu_show_cap(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) return sprintf(buf, "%llx\n", iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795) static ssize_t intel_iommu_show_ecap(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) return sprintf(buf, "%llx\n", iommu->ecap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) static ssize_t intel_iommu_show_ndoms(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) struct intel_iommu *iommu = dev_to_intel_iommu(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) cap_ndoms(iommu->cap)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) static struct attribute *intel_iommu_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824) &dev_attr_version.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) &dev_attr_address.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) &dev_attr_cap.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) &dev_attr_ecap.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) &dev_attr_domains_supported.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) &dev_attr_domains_used.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) static struct attribute_group intel_iommu_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) .name = "intel-iommu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) .attrs = intel_iommu_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) const struct attribute_group *intel_iommu_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) &intel_iommu_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) static inline bool has_external_pci(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) struct pci_dev *pdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) for_each_pci_dev(pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) if (pdev->external_facing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) static int __init platform_optin_force_iommu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) if (no_iommu || dmar_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) pr_info("Intel-IOMMU force enabled due to platform opt in\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) * If Intel-IOMMU is disabled by default, we will apply identity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) * map for all devices except those marked as being untrusted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) if (dmar_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) iommu_set_default_passthrough(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) dmar_disabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) no_iommu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875) static int __init probe_acpi_namespace_devices(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) /* To avoid a -Wunused-but-set-variable warning. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) struct intel_iommu *iommu __maybe_unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) int i, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) for_each_active_dev_scope(drhd->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) drhd->devices_cnt, i, dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) struct acpi_device_physical_node *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) struct iommu_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) struct acpi_device *adev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) if (dev->bus != &acpi_bus_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) adev = to_acpi_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) mutex_lock(&adev->physical_node_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) list_for_each_entry(pn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896) &adev->physical_node_list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) group = iommu_group_get(pn->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) if (group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) iommu_group_put(group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) pn->dev->bus->iommu_ops = &intel_iommu_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) ret = iommu_probe_device(pn->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) mutex_unlock(&adev->physical_node_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) int __init intel_iommu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) int ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) * Intel IOMMU is required for a TXT/tboot launch or platform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) * opt in, so enforce that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) platform_optin_force_iommu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931) if (iommu_init_mempool()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) panic("tboot: Failed to initialize iommu memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) down_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) if (dmar_table_init()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) panic("tboot: Failed to initialize DMAR table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) goto out_free_dmar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) if (dmar_dev_scope_init() < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) panic("tboot: Failed to initialize DMAR device scope\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) goto out_free_dmar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) up_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) * The bus notifier takes the dmar_global_lock, so lockdep will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) * complain later when we register it under the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) dmar_register_bus_notifier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) down_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) if (!no_iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) intel_iommu_debugfs_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) if (no_iommu || dmar_disabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) * We exit the function here to ensure IOMMU's remapping and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) * mempool aren't setup, which means that the IOMMU's PMRs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) * won't be disabled via the call to init_dmars(). So disable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) * it explicitly here. The PMRs were setup by tboot prior to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) * calling SENTER, but the kernel is expected to reset/tear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) * down the PMRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) if (intel_iommu_tboot_noforce) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) for_each_iommu(iommu, drhd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) * Make sure the IOMMUs are switched off, even when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) * boot into a kexec kernel and the previous kernel left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) * them enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) intel_disable_iommus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) goto out_free_dmar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) if (list_empty(&dmar_rmrr_units))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) pr_info("No RMRR found\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) if (list_empty(&dmar_atsr_units))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) pr_info("No ATSR found\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) if (dmar_init_reserved_ranges()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) panic("tboot: Failed to reserve iommu ranges\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) goto out_free_reserved_range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) if (dmar_map_gfx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) intel_iommu_gfx_mapped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) init_no_remapping_devices();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) ret = init_dmars();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) if (force_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) panic("tboot: Failed to initialize DMARs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) pr_err("Initialization failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) goto out_free_reserved_range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) up_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) init_iommu_pm_ops();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) down_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016) iommu_device_sysfs_add(&iommu->iommu, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) intel_iommu_groups,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) "%s", iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) iommu_device_register(&iommu->iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) up_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) if (si_domain && !hw_pass_through)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) register_memory_notifier(&intel_iommu_memory_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) intel_iommu_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) down_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) if (probe_acpi_namespace_devices())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) pr_warn("ACPI name space devices didn't probe correctly\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) /* Finally, we enable the DMA remapping hardware. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) for_each_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) if (!drhd->ignored && !translation_pre_enabled(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) iommu_enable_translation(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) iommu_disable_protect_mem_regions(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) up_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) intel_iommu_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) out_free_reserved_range:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) put_iova_domain(&reserved_iova_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) out_free_dmar:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) intel_iommu_free_dmars();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) up_write(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) iommu_exit_mempool();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) struct intel_iommu *iommu = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) * NB - intel-iommu lacks any sort of reference counting for the users of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) * dependent devices. If multiple endpoints have intersecting dependent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) * devices, unbinding the driver from any one of them will possibly leave
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) * the others unable to operate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) if (!iommu || !dev || !dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080) static void __dmar_remove_one_dev_info(struct device_domain_info *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) if (WARN_ON(!info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) iommu = info->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) domain = info->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) if (info->dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) if (dev_is_pci(info->dev) && sm_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) intel_pasid_tear_down_entry(iommu, info->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) PASID_RID2PASID, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) iommu_disable_dev_iotlb(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) if (!dev_is_real_dma_subdevice(info->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) domain_context_clear(iommu, info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) intel_pasid_free_table(info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) unlink_domain_info(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) spin_lock_irqsave(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) domain_detach_iommu(domain, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) spin_unlock_irqrestore(&iommu->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) free_devinfo_mem(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) static void dmar_remove_one_dev_info(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) if (info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) __dmar_remove_one_dev_info(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) static int md_domain_init(struct dmar_domain *domain, int guest_width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) int adjust_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) /* calculate AGAW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) domain->gaw = guest_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) adjust_width = guestwidth_to_adjustwidth(guest_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) domain->agaw = width_to_agaw(adjust_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) domain->iommu_coherency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) domain->iommu_snooping = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) domain->iommu_superpage = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) domain->max_addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) /* always allocate the top pgd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) if (!domain->pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) static void intel_init_iova_domain(struct dmar_domain *dmar_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150) init_iova_domain(&dmar_domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) copy_reserved_iova(&reserved_iova_list, &dmar_domain->iovad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) if (!intel_iommu_strict &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) init_iova_flush_queue(&dmar_domain->iovad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) iommu_flush_iova, iova_entry_free))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) pr_info("iova flush queue initialization failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) struct dmar_domain *dmar_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) struct iommu_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) case IOMMU_DOMAIN_DMA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) case IOMMU_DOMAIN_UNMANAGED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) dmar_domain = alloc_domain(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) if (!dmar_domain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) pr_err("Can't allocate dmar_domain\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) pr_err("Domain initialization failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174) domain_exit(dmar_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) if (type == IOMMU_DOMAIN_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) intel_init_iova_domain(dmar_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) domain = &dmar_domain->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) domain->geometry.aperture_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) domain->geometry.aperture_end =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) __DOMAIN_MAX_ADDR(dmar_domain->gaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) domain->geometry.force_aperture = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187) return domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) case IOMMU_DOMAIN_IDENTITY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) return &si_domain->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) static void intel_iommu_domain_free(struct iommu_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) if (domain != &si_domain->domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) domain_exit(to_dmar_domain(domain));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) * Check whether a @domain could be attached to the @dev through the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) * aux-domain attach/detach APIs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) static inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) is_aux_domain(struct device *dev, struct iommu_domain *domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) return info && info->auxd_enabled &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) domain->type == IOMMU_DOMAIN_UNMANAGED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) static void auxiliary_link_device(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) if (WARN_ON(!info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) domain->auxd_refcnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226) list_add(&domain->auxd, &info->auxiliary_domains);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) static void auxiliary_unlink_device(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) assert_spin_locked(&device_domain_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) if (WARN_ON(!info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) list_del(&domain->auxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) domain->auxd_refcnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) if (!domain->auxd_refcnt && domain->default_pasid > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) ioasid_free(domain->default_pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) static int aux_domain_add_dev(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) if (domain->default_pasid <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) u32 pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) /* No private data needed for the default pasid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) pasid = ioasid_alloc(NULL, PASID_MIN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) pci_max_pasids(to_pci_dev(dev)) - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) if (pasid == INVALID_IOASID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264) pr_err("Can't allocate default pasid\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) domain->default_pasid = pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) * iommu->lock must be held to attach domain to iommu and setup the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) * pasid entry for second level translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) ret = domain_attach_iommu(domain, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) goto attach_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) /* Setup the PASID entry for mediated devices: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) if (domain_use_first_level(domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) ret = domain_setup_first_level(iommu, domain, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) domain->default_pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) ret = intel_pasid_setup_second_level(iommu, domain, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) domain->default_pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) goto table_failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) auxiliary_link_device(domain, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) table_failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) domain_detach_iommu(domain, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) attach_failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) if (!domain->auxd_refcnt && domain->default_pasid > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) ioasid_free(domain->default_pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) static void aux_domain_remove_dev(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) if (!is_aux_domain(dev, &domain->domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) iommu = info->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) auxiliary_unlink_device(domain, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) domain_detach_iommu(domain, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) static int prepare_domain_attach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) int addr_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) /* check if this iommu agaw is sufficient for max mapped address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) addr_width = agaw_to_width(iommu->agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) if (addr_width > cap_mgaw(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) addr_width = cap_mgaw(iommu->cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) if (dmar_domain->max_addr > (1LL << addr_width)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) dev_err(dev, "%s: iommu width (%d) is not "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) "sufficient for the mapped address (%llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) __func__, addr_width, dmar_domain->max_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) dmar_domain->gaw = addr_width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357) * Knock out extra levels of page tables if necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) while (iommu->agaw < dmar_domain->agaw) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) struct dma_pte *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) pte = dmar_domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) if (dma_pte_present(pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) dmar_domain->pgd = (struct dma_pte *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) phys_to_virt(dma_pte_addr(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) free_pgtable_page(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) dmar_domain->agaw--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) static int intel_iommu_attach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) device_is_rmrr_locked(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) if (is_aux_domain(dev, domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) /* normally dev is not mapped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389) if (unlikely(domain_context_mapped(dev))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) struct dmar_domain *old_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392) old_domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) if (old_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) ret = prepare_domain_attach_device(domain, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) return domain_add_dev_info(to_dmar_domain(domain), dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) if (!is_aux_domain(dev, domain))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) ret = prepare_domain_attach_device(domain, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) return aux_domain_add_dev(to_dmar_domain(domain), dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) static void intel_iommu_detach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) aux_domain_remove_dev(to_dmar_domain(domain), dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433) * 2D array for converting and sanitizing IOMMU generic TLB granularity to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) * VT-d granularity. Invalidation is typically included in the unmap operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435) * as a result of DMA or VFIO unmap. However, for assigned devices guest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436) * owns the first level page tables. Invalidations of translation caches in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) * guest are trapped and passed down to the host.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439) * vIOMMU in the guest will only expose first level page tables, therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) * we do not support IOTLB granularity for request without PASID (second level).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442) * For example, to find the VT-d granularity encoding for IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) * type and page selective granularity within PASID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) * X: indexed by iommu cache type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445) * Y: indexed by enum iommu_inv_granularity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446) * [IOMMU_CACHE_INV_TYPE_IOTLB][IOMMU_INV_GRANU_ADDR]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449) static const int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450) inv_type_granu_table[IOMMU_CACHE_INV_TYPE_NR][IOMMU_INV_GRANU_NR] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) * PASID based IOTLB invalidation: PASID selective (per PASID),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453) * page selective (address granularity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) {-EINVAL, QI_GRAN_NONG_PASID, QI_GRAN_PSI_PASID},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) /* PASID based dev TLBs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457) {-EINVAL, -EINVAL, QI_DEV_IOTLB_GRAN_PASID_SEL},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) /* PASID cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459) {-EINVAL, -EINVAL, -EINVAL}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462) static inline int to_vtd_granularity(int type, int granu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) return inv_type_granu_table[type][granu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) static inline u64 to_vtd_size(u64 granu_size, u64 nr_granules)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469) u64 nr_pages = (granu_size * nr_granules) >> VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) /* VT-d size is encoded as 2^size of 4K pages, 0 for 4k, 9 for 2MB, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472) * IOMMU cache invalidate API passes granu_size in bytes, and number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) * granu size in contiguous memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475) return order_base_2(nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479) intel_iommu_sva_invalidate(struct iommu_domain *domain, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) struct iommu_cache_invalidate_info *inv_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486) int cache_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487) u8 bus, devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) u16 did, sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490) u64 size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) if (!inv_info || !dmar_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495) if (!dev || !dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498) iommu = device_to_iommu(dev, &bus, &devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) if (!(dmar_domain->flags & DOMAIN_FLAG_NESTING_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512) did = dmar_domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) sid = PCI_DEVID(bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) /* Size is only valid in address selective invalidation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516) if (inv_info->granularity == IOMMU_INV_GRANU_ADDR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) size = to_vtd_size(inv_info->granu.addr_info.granule_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518) inv_info->granu.addr_info.nb_granules);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) for_each_set_bit(cache_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521) (unsigned long *)&inv_info->cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) IOMMU_CACHE_INV_TYPE_NR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523) int granu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524) u64 pasid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525) u64 addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527) granu = to_vtd_granularity(cache_type, inv_info->granularity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528) if (granu == -EINVAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) pr_err_ratelimited("Invalid cache type and granu combination %d/%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) cache_type, inv_info->granularity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) * PASID is stored in different locations based on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536) * granularity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538) if (inv_info->granularity == IOMMU_INV_GRANU_PASID &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539) (inv_info->granu.pasid_info.flags & IOMMU_INV_PASID_FLAGS_PASID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540) pasid = inv_info->granu.pasid_info.pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541) else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) (inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_PASID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543) pasid = inv_info->granu.addr_info.pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545) switch (BIT(cache_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) case IOMMU_CACHE_INV_TYPE_IOTLB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547) /* HW will ignore LSB bits based on address mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548) if (inv_info->granularity == IOMMU_INV_GRANU_ADDR &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549) size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550) (inv_info->granu.addr_info.addr & ((BIT(VTD_PAGE_SHIFT + size)) - 1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551) pr_err_ratelimited("User address not aligned, 0x%llx, size order %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) inv_info->granu.addr_info.addr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556) * If granu is PASID-selective, address is ignored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557) * We use npages = -1 to indicate that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) qi_flush_piotlb(iommu, did, pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) mm_to_dma_pfn(inv_info->granu.addr_info.addr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) (granu == QI_GRAN_NONG_PASID) ? -1 : 1 << size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) inv_info->granu.addr_info.flags & IOMMU_INV_ADDR_FLAGS_LEAF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) if (!info->ats_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) * Always flush device IOTLB if ATS is enabled. vIOMMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568) * in the guest may assume IOTLB flush is inclusive,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569) * which is more efficient.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) case IOMMU_CACHE_INV_TYPE_DEV_IOTLB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574) * PASID based device TLB invalidation does not support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575) * IOMMU_INV_GRANU_PASID granularity but only supports
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) * IOMMU_INV_GRANU_ADDR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577) * The equivalent of that is we set the size to be the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578) * entire range of 64 bit. User only provides PASID info
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579) * without address info. So we set addr to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) if (inv_info->granularity == IOMMU_INV_GRANU_PASID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) size = 64 - VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583) addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584) } else if (inv_info->granularity == IOMMU_INV_GRANU_ADDR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) addr = inv_info->granu.addr_info.addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588) if (info->ats_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589) qi_flush_dev_iotlb_pasid(iommu, sid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590) info->pfsid, pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) info->ats_qdep, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594) pr_warn_ratelimited("Passdown device IOTLB flush w/o ATS!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597) dev_err_ratelimited(dev, "Unsupported IOMMU invalidation type %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598) cache_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) static int intel_iommu_map(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611) unsigned long iova, phys_addr_t hpa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) size_t size, int iommu_prot, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615) u64 max_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616) int prot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619) if (iommu_prot & IOMMU_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) prot |= DMA_PTE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621) if (iommu_prot & IOMMU_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622) prot |= DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624) prot |= DMA_PTE_SNP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) max_addr = iova + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) if (dmar_domain->max_addr < max_addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628) u64 end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) /* check if minimum agaw is sufficient for mapped address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631) end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) if (end < max_addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) pr_err("%s: iommu width (%d) is not "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634) "sufficient for the mapped address (%llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) __func__, dmar_domain->gaw, max_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) dmar_domain->max_addr = max_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640) /* Round up size to next multiple of PAGE_SIZE, if it and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) the low bits of hpa would take us onto the next page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) size = aligned_nrpages(hpa, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643) ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644) hpa >> VTD_PAGE_SHIFT, size, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) static size_t intel_iommu_unmap(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) unsigned long iova, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650) struct iommu_iotlb_gather *gather)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653) struct page *freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654) unsigned long start_pfn, last_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655) unsigned int npages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) int iommu_id, level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658) /* Cope with horrid API which requires us to unmap more than the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659) size argument if it happens to be a large-page mapping. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662) if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663) size = VTD_PAGE_SIZE << level_to_offset_bits(level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665) start_pfn = iova >> VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666) last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670) npages = last_pfn - start_pfn + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672) for_each_domain_iommu(iommu_id, dmar_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) start_pfn, npages, !freelist, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676) dma_free_pagelist(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678) if (dmar_domain->max_addr == iova + size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) dmar_domain->max_addr = iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685) dma_addr_t iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) struct dma_pte *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689) int level = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) u64 phys = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692) pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) if (pte && dma_pte_present(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694) phys = dma_pte_addr(pte) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) (iova & (BIT_MASK(level_to_offset_bits(level) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696) VTD_PAGE_SHIFT) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) return phys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701) static inline bool scalable_mode_support(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) bool ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) if (!sm_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) static inline bool iommu_pasid_support(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) bool ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727) if (!pasid_supported(iommu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728) ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737) static inline bool nested_mode_support(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) struct dmar_drhd_unit *drhd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) bool ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744) for_each_active_iommu(iommu, drhd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745) if (!sm_supported(iommu) || !ecap_nest(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755) static bool intel_iommu_capable(enum iommu_cap cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) if (cap == IOMMU_CAP_CACHE_COHERENCY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) return domain_update_iommu_snooping(NULL) == 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759) if (cap == IOMMU_CAP_INTR_REMAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) return irq_remapping_enabled == 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765) static struct iommu_device *intel_iommu_probe_device(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771) return ERR_PTR(-ENODEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) if (translation_pre_enabled(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776) return &iommu->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779) static void intel_iommu_release_device(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787) dmar_remove_one_dev_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789) set_dma_ops(dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) static void intel_iommu_probe_finalize(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794) struct iommu_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) domain = iommu_get_domain_for_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) if (device_needs_bounce(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798) set_dma_ops(dev, &bounce_dma_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) else if (domain && domain->type == IOMMU_DOMAIN_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) set_dma_ops(dev, &intel_dma_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802) set_dma_ops(dev, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) static void intel_iommu_get_resv_regions(struct device *device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) struct list_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808) int prot = DMA_PTE_READ | DMA_PTE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809) struct iommu_resv_region *reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) struct dmar_rmrr_unit *rmrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) struct device *i_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814) down_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815) for_each_rmrr_units(rmrr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) i, i_dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818) struct iommu_resv_region *resv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) enum iommu_resv_type type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820) size_t length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) if (i_dev != device &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823) !is_downstream_to_pci_bridge(device, i_dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) length = rmrr->end_address - rmrr->base_address + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) type = device_rmrr_is_relaxable(device) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829) IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) resv = iommu_alloc_resv_region(rmrr->base_address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832) length, prot, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) if (!resv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836) list_add_tail(&resv->list, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839) up_read(&dmar_global_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) if (dev_is_pci(device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) struct pci_dev *pdev = to_pci_dev(device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847) IOMMU_RESV_DIRECT_RELAXABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) if (reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) list_add_tail(®->list, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854) reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856) 0, IOMMU_RESV_MSI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857) if (!reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) list_add_tail(®->list, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) struct context_entry *context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) u64 ctx_lo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871) domain = find_domain(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) if (!domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876) spin_lock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880) if (!info || !info->pasid_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883) context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) if (WARN_ON(!context))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) ctx_lo = context[0].lo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) if (!(ctx_lo & CONTEXT_PASIDE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) ctx_lo |= CONTEXT_PASIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891) context[0].lo = ctx_lo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) iommu->flush.flush_context(iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894) domain->iommu_did[iommu->seq_id],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895) PCI_DEVID(info->bus, info->devfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) DMA_CCMD_MASK_NOBIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) DMA_CCMD_DEVICE_INVL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900) /* Enable PASID support in the device, if it wasn't already */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) if (!info->pasid_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) iommu_enable_dev_iotlb(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) spin_unlock(&iommu->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913) static void intel_iommu_apply_resv_region(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915) struct iommu_resv_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918) unsigned long start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) start = IOVA_PFN(region->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921) end = IOVA_PFN(region->start + region->length - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) static struct iommu_group *intel_iommu_device_group(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) if (dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) return pci_device_group(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930) return generic_device_group(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) static int intel_iommu_enable_auxd(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5941) if (!iommu || dmar_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5942) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5944) if (!sm_supported(iommu) || !pasid_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5945) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5947) ret = intel_iommu_enable_pasid(iommu, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5948) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5949) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5951) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5952) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5953) info->auxd_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5954) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5956) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5959) static int intel_iommu_disable_auxd(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5961) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5962) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5964) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5965) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5966) if (!WARN_ON(!info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5967) info->auxd_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5968) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5970) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5973) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5974) * A PCI express designated vendor specific extended capability is defined
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5975) * in the section 3.7 of Intel scalable I/O virtualization technical spec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5976) * for system software and tools to detect endpoint devices supporting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5977) * Intel scalable IO virtualization without host driver dependency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5978) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5979) * Returns the address of the matching extended capability structure within
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5980) * the device's PCI configuration space or 0 if the device does not support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5981) * it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5983) static int siov_find_pci_dvsec(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5984) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5985) int pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5986) u16 vendor, id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5988) pos = pci_find_next_ext_capability(pdev, 0, 0x23);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5989) while (pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5990) pci_read_config_word(pdev, pos + 4, &vendor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5991) pci_read_config_word(pdev, pos + 8, &id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5992) if (vendor == PCI_VENDOR_ID_INTEL && id == 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5993) return pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5995) pos = pci_find_next_ext_capability(pdev, pos, 0x23);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5998) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6001) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6002) intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6003) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6004) if (feat == IOMMU_DEV_FEAT_AUX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6005) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6007) if (!dev_is_pci(dev) || dmar_disabled ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6008) !scalable_mode_support() || !iommu_pasid_support())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6009) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6011) ret = pci_pasid_features(to_pci_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6012) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6013) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6015) return !!siov_find_pci_dvsec(to_pci_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6016) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6018) if (feat == IOMMU_DEV_FEAT_SVA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6019) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6021) return info && (info->iommu->flags & VTD_FLAG_SVM_CAPABLE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6022) info->pasid_supported && info->pri_supported &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6023) info->ats_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6026) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6029) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6030) intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6032) if (feat == IOMMU_DEV_FEAT_AUX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6033) return intel_iommu_enable_auxd(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6035) if (feat == IOMMU_DEV_FEAT_SVA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6036) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6038) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6039) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6041) if (info->iommu->flags & VTD_FLAG_SVM_CAPABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6042) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6045) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6048) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6049) intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6050) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6051) if (feat == IOMMU_DEV_FEAT_AUX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6052) return intel_iommu_disable_auxd(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6054) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6057) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6058) intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6060) struct device_domain_info *info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6062) if (feat == IOMMU_DEV_FEAT_AUX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6063) return scalable_mode_support() && info && info->auxd_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6065) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6068) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6069) intel_iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6071) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6073) return dmar_domain->default_pasid > 0 ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6074) dmar_domain->default_pasid : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6075) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6077) static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6078) struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6080) return attach_deferred(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6083) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6084) intel_iommu_domain_set_attr(struct iommu_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6085) enum iommu_attr attr, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6087) struct dmar_domain *dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6088) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6089) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6091) if (domain->type != IOMMU_DOMAIN_UNMANAGED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6092) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6094) switch (attr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6095) case DOMAIN_ATTR_NESTING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6096) spin_lock_irqsave(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6097) if (nested_mode_support() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6098) list_empty(&dmar_domain->devices)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6099) dmar_domain->flags |= DOMAIN_FLAG_NESTING_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6100) dmar_domain->flags &= ~DOMAIN_FLAG_USE_FIRST_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6101) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6102) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6104) spin_unlock_irqrestore(&device_domain_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6105) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6106) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6107) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6108) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6111) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6115) * Check that the device does not live on an external facing PCI port that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6116) * marked as untrusted. Such devices should not be able to apply quirks and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6117) * thus not be able to bypass the IOMMU restrictions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6119) static bool risky_device(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6121) if (pdev->untrusted) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6122) pci_info(pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6123) "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6124) pdev->vendor, pdev->device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6125) pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6126) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6128) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6131) const struct iommu_ops intel_iommu_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6132) .capable = intel_iommu_capable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6133) .domain_alloc = intel_iommu_domain_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6134) .domain_free = intel_iommu_domain_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6135) .domain_set_attr = intel_iommu_domain_set_attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6136) .attach_dev = intel_iommu_attach_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6137) .detach_dev = intel_iommu_detach_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6138) .aux_attach_dev = intel_iommu_aux_attach_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6139) .aux_detach_dev = intel_iommu_aux_detach_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6140) .aux_get_pasid = intel_iommu_aux_get_pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6141) .map = intel_iommu_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6142) .unmap = intel_iommu_unmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6143) .iova_to_phys = intel_iommu_iova_to_phys,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6144) .probe_device = intel_iommu_probe_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6145) .probe_finalize = intel_iommu_probe_finalize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6146) .release_device = intel_iommu_release_device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6147) .get_resv_regions = intel_iommu_get_resv_regions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6148) .put_resv_regions = generic_iommu_put_resv_regions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6149) .apply_resv_region = intel_iommu_apply_resv_region,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6150) .device_group = intel_iommu_device_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6151) .dev_has_feat = intel_iommu_dev_has_feat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6152) .dev_feat_enabled = intel_iommu_dev_feat_enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6153) .dev_enable_feat = intel_iommu_dev_enable_feat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6154) .dev_disable_feat = intel_iommu_dev_disable_feat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6155) .is_attach_deferred = intel_iommu_is_attach_deferred,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6156) .def_domain_type = device_def_domain_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6157) .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6158) #ifdef CONFIG_INTEL_IOMMU_SVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6159) .cache_invalidate = intel_iommu_sva_invalidate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6160) .sva_bind_gpasid = intel_svm_bind_gpasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6161) .sva_unbind_gpasid = intel_svm_unbind_gpasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6162) .sva_bind = intel_svm_bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6163) .sva_unbind = intel_svm_unbind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6164) .sva_get_pasid = intel_svm_get_pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6165) .page_response = intel_svm_page_response,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6166) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6167) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6169) static void quirk_iommu_igfx(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6171) if (risky_device(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6172) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6174) pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6175) dmar_map_gfx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6178) /* G4x/GM45 integrated gfx dmar support is totally busted. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6179) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6180) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6181) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6182) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6183) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6184) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6185) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6187) /* Broadwell igfx malfunctions with dmar */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6188) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6189) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6190) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6191) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6192) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6193) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6194) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6195) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6196) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6197) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6198) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6199) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6200) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6201) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6202) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6203) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6204) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6205) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6206) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6207) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6208) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6209) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6210) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6211) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6213) static void quirk_iommu_rwbf(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6215) if (risky_device(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6216) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6219) * Mobile 4 Series Chipset neglects to set RWBF capability,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6220) * but needs it. Same seems to hold for the desktop versions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6222) pci_info(dev, "Forcing write-buffer flush capability\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6223) rwbf_quirk = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6226) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6227) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6228) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6229) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6230) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6231) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6232) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6234) #define GGC 0x52
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6235) #define GGC_MEMORY_SIZE_MASK (0xf << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6236) #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6237) #define GGC_MEMORY_SIZE_1M (0x1 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6238) #define GGC_MEMORY_SIZE_2M (0x3 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6239) #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6240) #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6241) #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6242) #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6244) static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6246) unsigned short ggc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6248) if (risky_device(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6249) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6251) if (pci_read_config_word(dev, GGC, &ggc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6252) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6254) if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6255) pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6256) dmar_map_gfx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6257) } else if (dmar_map_gfx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6258) /* we have to ensure the gfx device is idle before we flush */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6259) pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6260) intel_iommu_strict = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6263) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6264) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6265) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6266) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6268) static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6270) unsigned short ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6272) if (!IS_GFX_DEVICE(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6273) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6275) ver = (dev->device >> 8) & 0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6276) if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6277) ver != 0x4e && ver != 0x8a && ver != 0x98 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6278) ver != 0x9a)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6279) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6281) if (risky_device(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6282) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6284) pci_info(dev, "Skip IOMMU disabling for graphics\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6285) iommu_skip_te_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6287) DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6289) /* On Tylersburg chipsets, some BIOSes have been known to enable the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6290) ISOCH DMAR unit for the Azalia sound device, but not give it any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6291) TLB entries, which causes it to deadlock. Check for that. We do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6292) this in a function called from init_dmars(), instead of in a PCI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6293) quirk, because we don't want to print the obnoxious "BIOS broken"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6294) message if VT-d is actually disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6296) static void __init check_tylersburg_isoch(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6298) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6299) uint32_t vtisochctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6301) /* If there's no Azalia in the system anyway, forget it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6302) pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6303) if (!pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6304) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6306) if (risky_device(pdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6307) pci_dev_put(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6308) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6311) pci_dev_put(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6313) /* System Management Registers. Might be hidden, in which case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6314) we can't do the sanity check. But that's OK, because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6315) known-broken BIOSes _don't_ actually hide it, so far. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6316) pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6317) if (!pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6318) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6320) if (risky_device(pdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6321) pci_dev_put(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6322) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6325) if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6326) pci_dev_put(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6327) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6330) pci_dev_put(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6332) /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6333) if (vtisochctrl & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6334) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6336) /* Drop all bits other than the number of TLB entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6337) vtisochctrl &= 0x1c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6339) /* If we have the recommended number of TLB entries (16), fine. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6340) if (vtisochctrl == 0x10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6341) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6343) /* Zero TLB entries? You get to ride the short bus to school. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6344) if (!vtisochctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6345) WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6346) "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6347) dmi_get_system_info(DMI_BIOS_VENDOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6348) dmi_get_system_info(DMI_BIOS_VERSION),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6349) dmi_get_system_info(DMI_PRODUCT_VERSION));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6350) iommu_identity_mapping |= IDENTMAP_AZALIA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6351) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6354) pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6355) vtisochctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6356) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate