^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright © 2015 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Authors: David Woodhouse <dwmw2@infradead.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/intel-iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/mmu_notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/intel-svm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/pci-ats.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/dmar.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mm_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/ioasid.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/fpu/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "pasid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static irqreturn_t prq_event_thread(int irq, void *d);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static void intel_svm_drain_prq(struct device *dev, u32 pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define PRQ_ORDER 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int intel_svm_enable_prq(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct page *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int irq, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (!pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) pr_warn("IOMMU: %s: Failed to allocate page request queue\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) iommu->prq = page_address(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) if (irq <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) free_pages((unsigned long)iommu->prq, PRQ_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) iommu->prq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) iommu->pr_irq = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) iommu->prq_name, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) dmar_free_hwirq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) iommu->pr_irq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) init_completion(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) int intel_svm_finish_prq(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (iommu->pr_irq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) free_irq(iommu->pr_irq, iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) dmar_free_hwirq(iommu->pr_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) iommu->pr_irq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) free_pages((unsigned long)iommu->prq, PRQ_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) iommu->prq = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static inline bool intel_svm_capable(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return iommu->flags & VTD_FLAG_SVM_CAPABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) void intel_svm_check(struct intel_iommu *iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (!pasid_supported(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) !cap_fl1gp_support(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) pr_err("%s SVM disabled, incompatible 1GB page capability\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (cpu_feature_enabled(X86_FEATURE_LA57) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) !cap_5lp_support(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) pr_err("%s SVM disabled, incompatible paging mode\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) iommu->flags |= VTD_FLAG_SVM_CAPABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void __flush_svm_range_dev(struct intel_svm *svm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct intel_svm_dev *sdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) unsigned long pages, int ih)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct device_domain_info *info = get_domain_info(sdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (WARN_ON(!pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (info->ats_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) svm->pasid, sdev->qdep, address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) order_base_2(pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static void intel_flush_svm_range_dev(struct intel_svm *svm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct intel_svm_dev *sdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) unsigned long pages, int ih)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned long shift = ilog2(__roundup_pow_of_two(pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned long start = ALIGN_DOWN(address, align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) while (start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) start += align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned long pages, int ih)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) list_for_each_entry_rcu(sdev, &svm->devs, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) intel_flush_svm_range_dev(svm, sdev, address, pages, ih);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* Pages have been freed at this point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void intel_invalidate_range(struct mmu_notifier *mn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) intel_flush_svm_range(svm, start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /* This might end up being called from exit_mmap(), *before* the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * tables are cleared. And __mmu_notifier_release() will delete us from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * the list of notifiers so that our invalidate_range() callback doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * get called when the page tables are cleared. So we need to protect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * against hardware accessing those page tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * We do it by clearing the entry in the PASID table and then flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * the IOTLB and the PASID table caches. This might upset hardware;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * perhaps we'll want to point the PASID to a dummy PGD (like the zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * page) so that we end up taking a fault that the hardware really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * *has* to handle gracefully without affecting other processes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) list_for_each_entry_rcu(sdev, &svm->devs, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) intel_pasid_tear_down_entry(sdev->iommu, sdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) svm->pasid, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static const struct mmu_notifier_ops intel_mmuops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) .release = intel_mm_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) .invalidate_range = intel_invalidate_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static DEFINE_MUTEX(pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static LIST_HEAD(global_svm_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #define for_each_svm_dev(sdev, svm, d) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) list_for_each_entry((sdev), &(svm)->devs, list) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if ((d) != (sdev)->dev) {} else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct intel_svm **rsvm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct intel_svm_dev **rsdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct intel_svm_dev *d, *sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct intel_svm *svm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /* The caller should hold the pasid_mutex lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (WARN_ON(!mutex_is_locked(&pasid_mutex)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (pasid == INVALID_IOASID || pasid >= PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) svm = ioasid_find(NULL, pasid, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (IS_ERR(svm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return PTR_ERR(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (!svm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * If we found svm for the PASID, there must be at least one device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * bond.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (WARN_ON(list_empty(&svm->devs)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) list_for_each_entry_rcu(d, &svm->devs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (d->dev == dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) sdev = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) *rsvm = svm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) *rsdev = sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct iommu_gpasid_bind_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct intel_svm_dev *sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct dmar_domain *dmar_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct intel_svm *svm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) unsigned long iflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (WARN_ON(!iommu) || !data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /* IOMMU core ensures argsz is more than the start of the union */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /* Make sure no undefined flags are used in vendor data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (!dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* VT-d supports devices with full 20 bit PASIDs only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * We only check host PASID range, we have no knowledge to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * guest PASID range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (data->hpasid <= 0 || data->hpasid >= PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) dmar_domain = to_dmar_domain(domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * Do not allow multiple bindings of the same device-PASID since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * there is only one SL page tables per PASID. We may revisit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * once sharing PGD across domains are supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) dev_warn_ratelimited(dev, "Already bound with PASID %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (!svm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* We come here when PASID has never been bond to a device. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) svm = kzalloc(sizeof(*svm), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (!svm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* REVISIT: upper layer/VFIO can track host process that bind
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * the PASID. ioasid_set = mm might be sufficient for vfio to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * check pasid VMM ownership. We can drop the following line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) * once VFIO and IOASID set check is in place.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) svm->mm = get_task_mm(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) svm->pasid = data->hpasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (data->flags & IOMMU_SVA_GPASID_VAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) svm->gpasid = data->gpasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) svm->flags |= SVM_FLAG_GUEST_PASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) ioasid_set_data(data->hpasid, svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) INIT_LIST_HEAD_RCU(&svm->devs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) mmput(svm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (!sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) sdev->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) sdev->sid = PCI_DEVID(info->bus, info->devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) sdev->iommu = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /* Only count users if device has aux domains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) sdev->users = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* Set up device context entry for PASID if not enabled already */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) ret = intel_iommu_enable_pasid(iommu, sdev->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) dev_err_ratelimited(dev, "Failed to enable PASID capability\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * PASID table is per device for better security. Therefore, for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * each bind of a new device even with an existing PASID, we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * call the nested mode setup function here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) spin_lock_irqsave(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) ret = intel_pasid_setup_nested(iommu, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) (pgd_t *)(uintptr_t)data->gpgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) data->hpasid, &data->vendor.vtd, dmar_domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) data->addr_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) spin_unlock_irqrestore(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) data->hpasid, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * PASID entry should be in cleared state if nested mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * set up failed. So we only need to clear IOASID tracking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * data such that free call will succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) svm->flags |= SVM_FLAG_GUEST_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) init_rcu_head(&sdev->rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) list_add_rcu(&sdev->list, &svm->devs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) ioasid_set_data(data->hpasid, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) struct intel_svm *svm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (WARN_ON(!iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) sdev->users--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (!sdev->users) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) list_del_rcu(&sdev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) intel_pasid_tear_down_entry(iommu, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) svm->pasid, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) intel_svm_drain_prq(dev, svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) kfree_rcu(sdev, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (list_empty(&svm->devs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * We do not free the IOASID here in that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * IOMMU driver did not allocate it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * Unlike native SVM, IOASID for guest use was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * allocated prior to the bind call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * In any case, if the free call comes before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * the unbind, IOMMU driver will get notified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * and perform cleanup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ioasid_set_data(pasid, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) static void _load_pasid(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) update_pasid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static void load_pasid(struct mm_struct *mm, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) mutex_lock(&mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /* Synchronize with READ_ONCE in update_pasid(). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) smp_store_release(&mm->pasid, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* Update PASID MSR on all CPUs running the mm's tasks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) on_each_cpu_mask(mm_cpumask(mm), _load_pasid, NULL, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) mutex_unlock(&mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /* Caller must hold pasid_mutex, mm reference */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) intel_svm_bind_mm(struct device *dev, unsigned int flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct svm_dev_ops *ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct mm_struct *mm, struct intel_svm_dev **sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) struct intel_svm *svm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) unsigned long iflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) int pasid_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (!iommu || dmar_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (!intel_svm_capable(iommu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (dev_is_pci(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) pasid_max = pci_max_pasids(to_pci_dev(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (pasid_max < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) pasid_max = 1 << 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /* Bind supervisor PASID shuld have mm = NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (flags & SVM_FLAG_SUPERVISOR_MODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (!ecap_srs(iommu->ecap) || mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) pr_err("Supervisor PASID with user provided mm.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (!(flags & SVM_FLAG_PRIVATE_PASID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct intel_svm *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) list_for_each_entry(t, &global_svm_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) svm = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (svm->pasid >= pasid_max) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) dev_warn(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) "Limited PASID width. Cannot use existing PASID %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /* Find the matching device in svm list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) for_each_svm_dev(sdev, svm, dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (sdev->ops != ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) sdev->users++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) goto success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (!sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) sdev->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) sdev->iommu = iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) ret = intel_iommu_enable_pasid(iommu, dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) sdev->did = FLPT_DEFAULT_DID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) sdev->sid = PCI_DEVID(info->bus, info->devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (info->ats_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) sdev->dev_iotlb = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) sdev->qdep = info->ats_qdep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) sdev->qdep = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) /* Finish the setup now we know we're keeping it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) sdev->users = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) sdev->ops = ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) init_rcu_head(&sdev->rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (!svm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) svm = kzalloc(sizeof(*svm), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (!svm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (pasid_max > intel_pasid_max_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) pasid_max = intel_pasid_max_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /* Do not use PASID 0, reserved for RID to PASID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) svm->pasid = ioasid_alloc(NULL, PASID_MIN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) pasid_max - 1, svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (svm->pasid == INVALID_IOASID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) svm->notifier.ops = &intel_mmuops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) svm->mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) svm->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) INIT_LIST_HEAD_RCU(&svm->devs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) INIT_LIST_HEAD(&svm->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) ret = mmu_notifier_register(&svm->notifier, mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) ioasid_free(svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) spin_lock_irqsave(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) ret = intel_pasid_setup_first_level(iommu, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) mm ? mm->pgd : init_mm.pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) svm->pasid, FLPT_DEFAULT_DID,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) (cpu_feature_enabled(X86_FEATURE_LA57) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) PASID_FLAG_FL5LP : 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) spin_unlock_irqrestore(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) mmu_notifier_unregister(&svm->notifier, mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ioasid_free(svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) list_add_tail(&svm->list, &global_svm_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* The newly allocated pasid is loaded to the mm. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) load_pasid(mm, svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * Binding a new device with existing PASID, need to setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * the PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) spin_lock_irqsave(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) ret = intel_pasid_setup_first_level(iommu, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) mm ? mm->pgd : init_mm.pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) svm->pasid, FLPT_DEFAULT_DID,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) (cpu_feature_enabled(X86_FEATURE_LA57) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) PASID_FLAG_FL5LP : 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) spin_unlock_irqrestore(&iommu->lock, iflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) kfree(sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) list_add_rcu(&sdev->list, &svm->devs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) success:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) sdev->pasid = svm->pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) sdev->sva.dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) *sd = sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* Caller must hold pasid_mutex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static int intel_svm_unbind_mm(struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct intel_svm *svm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) iommu = device_to_iommu(dev, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) sdev->users--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (!sdev->users) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) list_del_rcu(&sdev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* Flush the PASID cache and IOTLB for this device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * Note that we do depend on the hardware *not* using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * the PASID any more. Just as we depend on other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * devices never using PASIDs that they have no right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * to use. We have a *shared* PASID table, because it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * large and has to be physically contiguous. So it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * hard to be as defensive as we might like. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) intel_pasid_tear_down_entry(iommu, dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) svm->pasid, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) intel_svm_drain_prq(dev, svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) kfree_rcu(sdev, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (list_empty(&svm->devs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ioasid_free(svm->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (svm->mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) mmu_notifier_unregister(&svm->notifier, svm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /* Clear mm's pasid. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) load_pasid(svm->mm, PASID_DISABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) list_del(&svm->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* We mandate that no page faults may be outstanding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * for the PASID when intel_svm_unbind_mm() is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * If that is not obeyed, subtle errors will happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * Let's make them less subtle... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) memset(svm, 0x6b, sizeof(*svm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) kfree(svm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /* Page request queue descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct page_req_dsc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) u64 type:8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) u64 pasid_present:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) u64 priv_data_present:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) u64 rsvd:6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) u64 rid:16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) u64 pasid:20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) u64 exe_req:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) u64 pm_req:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) u64 rsvd2:10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) u64 qw_0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) u64 rd_req:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) u64 wr_req:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) u64 lpig:1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) u64 prg_index:9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) u64 addr:52;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) u64 qw_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) u64 priv_data[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) unsigned long requested = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (req->exe_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) requested |= VM_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (req->rd_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) requested |= VM_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (req->wr_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) requested |= VM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) return (requested & ~vma->vm_flags) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) static bool is_canonical_address(u64 addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) long saddr = (long) addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) return (((saddr << shift) >> shift) == saddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * intel_svm_drain_prq - Drain page requests and responses for a pasid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * @dev: target device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * @pasid: pasid for draining
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * Drain all pending page requests and responses related to @pasid in both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * software and hardware. This is supposed to be called after the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * and DevTLB have been invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * It waits until all pending page requests for @pasid in the page fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * queue are completed by the prq handling thread. Then follow the steps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * described in VT-d spec CH7.10 to drain all page requests and page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * responses pending in the hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) static void intel_svm_drain_prq(struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) struct dmar_domain *domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) struct qi_desc desc[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) struct pci_dev *pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) int head, tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) u16 sid, did;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) int qdep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (WARN_ON(!info || !dev_is_pci(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (!info->pri_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) iommu = info->iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) domain = info->domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) sid = PCI_DEVID(info->bus, info->devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) qdep = pci_ats_queue_depth(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * Check and wait until all pending page requests in the queue are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * handled by the prq handling thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) prq_retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) reinit_completion(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) while (head != tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) struct page_req_dsc *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) req = &iommu->prq[head / sizeof(*req)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) if (!req->pasid_present || req->pasid != pasid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) head = (head + sizeof(*req)) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) wait_for_completion(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) goto prq_retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * Perform steps described in VT-d spec CH7.10 to drain page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * requests and responses in hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) memset(desc, 0, sizeof(desc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) QI_IWD_FENCE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) QI_IWD_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) desc[1].qw0 = QI_EIOTLB_PASID(pasid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) QI_EIOTLB_DID(did) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) QI_EIOTLB_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) QI_DEV_EIOTLB_SID(sid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) QI_DEV_EIOTLB_QDEP(qdep) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) QI_DEIOTLB_TYPE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) QI_DEV_IOTLB_PFSID(info->pfsid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) qi_retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) reinit_completion(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) wait_for_completion(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) goto qi_retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) static int prq_to_iommu_prot(struct page_req_dsc *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) int prot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (req->rd_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) prot |= IOMMU_FAULT_PERM_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (req->wr_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) prot |= IOMMU_FAULT_PERM_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) if (req->exe_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) prot |= IOMMU_FAULT_PERM_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (req->pm_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) prot |= IOMMU_FAULT_PERM_PRIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) return prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) struct iommu_fault_event event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) if (!dev || !dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) /* Fill in event data for device specific processing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) memset(&event, 0, sizeof(struct iommu_fault_event));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) event.fault.type = IOMMU_FAULT_PAGE_REQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) event.fault.prm.addr = (u64)desc->addr << VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) event.fault.prm.pasid = desc->pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) event.fault.prm.grpid = desc->prg_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) event.fault.prm.perm = prq_to_iommu_prot(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (desc->lpig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) if (desc->pasid_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (desc->priv_data_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * Set last page in group bit if private data is present,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * page response is required as it does for LPIG.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * iommu_report_device_fault() doesn't understand this vendor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * specific requirement thus we set last_page as a workaround.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) memcpy(event.fault.prm.private_data, desc->priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) sizeof(desc->priv_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) return iommu_report_device_fault(dev, &event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) static irqreturn_t prq_event_thread(int irq, void *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) struct intel_svm_dev *sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) struct intel_iommu *iommu = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) struct intel_svm *svm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) int head, tail, handled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) /* Clear PPR bit before reading head/tail registers, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * ensure that we get a new interrupt if needed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) while (head != tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) struct page_req_dsc *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) struct qi_desc resp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) vm_fault_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) u64 address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) handled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) req = &iommu->prq[head / sizeof(*req)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) result = QI_RESP_FAILURE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) address = (u64)req->addr << VTD_PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) if (!req->pasid_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) pr_err("%s: Page request without PASID: %08llx %08llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) iommu->name, ((unsigned long long *)req)[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) ((unsigned long long *)req)[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) goto no_pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /* We shall not receive page request for supervisor SVM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) if (req->pm_req && (req->rd_req | req->wr_req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) pr_err("Unexpected page request in Privilege Mode");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) /* No need to find the matching sdev as for bad_req */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) goto no_pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /* DMA read with exec requeset is not supported. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (req->exe_req && req->rd_req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) pr_err("Execution request not supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) goto no_pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) if (!svm || svm->pasid != req->pasid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) svm = ioasid_find(NULL, req->pasid, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /* It *can't* go away, because the driver is not permitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * to unbind the mm while any page faults are outstanding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * So we only need RCU to protect the internal idr code. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (IS_ERR_OR_NULL(svm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) iommu->name, req->pasid, ((unsigned long long *)req)[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) ((unsigned long long *)req)[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) goto no_pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (!sdev || sdev->sid != req->rid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) struct intel_svm_dev *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) list_for_each_entry_rcu(t, &svm->devs, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (t->sid == req->rid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) sdev = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) result = QI_RESP_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /* Since we're using init_mm.pgd directly, we should never take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * any faults on kernel addresses. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if (!svm->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) goto bad_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) /* If address is not canonical, return invalid response */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (!is_canonical_address(address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) goto bad_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * If prq is to be handled outside iommu driver via receiver of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * the fault notifiers, we skip the page response here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (svm->flags & SVM_FLAG_GUEST_MODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (sdev && !intel_svm_prq_report(sdev->dev, req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) goto prq_advance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) goto bad_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) /* If the mm is already defunct, don't handle faults. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (!mmget_not_zero(svm->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) goto bad_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) mmap_read_lock(svm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) vma = find_extend_vma(svm->mm, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (!vma || address < vma->vm_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) goto invalid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (access_error(vma, req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) goto invalid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) ret = handle_mm_fault(vma, address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) req->wr_req ? FAULT_FLAG_WRITE : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) if (ret & VM_FAULT_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) goto invalid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) result = QI_RESP_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) invalid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) mmap_read_unlock(svm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) mmput(svm->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) bad_req:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) WARN_ON(!sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) if (sdev && sdev->ops && sdev->ops->fault_cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) int rwxp = (req->rd_req << 3) | (req->wr_req << 2) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) (req->exe_req << 1) | (req->pm_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) req->priv_data, rwxp, result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) /* We get here in the error case where the PASID lookup failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) and these can be NULL. Do not use them below this point! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) svm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) no_pasid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (req->lpig || req->priv_data_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) * Per VT-d spec. v3.0 ch7.7, system software must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) * respond with page group response if private data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * is present (PDP) or last page in group (LPIG) bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) * is set. This is an additional VT-d feature beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * PCI ATS spec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) resp.qw0 = QI_PGRP_PASID(req->pasid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) QI_PGRP_DID(req->rid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) QI_PGRP_PASID_P(req->pasid_present) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) QI_PGRP_PDP(req->priv_data_present) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) QI_PGRP_RESP_CODE(result) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) QI_PGRP_RESP_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) resp.qw1 = QI_PGRP_IDX(req->prg_index) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) QI_PGRP_LPIG(req->lpig);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) resp.qw2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) resp.qw3 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) if (req->priv_data_present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) memcpy(&resp.qw2, req->priv_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) sizeof(req->priv_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) qi_submit_sync(iommu, &resp, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) prq_advance:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) head = (head + sizeof(*req)) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) dmar_writeq(iommu->reg + DMAR_PQH_REG, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * Clear the page request overflow bit and wake up all threads that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) * are waiting for the completion of this handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) pr_info_ratelimited("IOMMU: %s: PRQ overflow detected\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (head == tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) pr_info_ratelimited("IOMMU: %s: PRQ overflow cleared",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) if (!completion_done(&iommu->prq_complete))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) complete(&iommu->prq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return IRQ_RETVAL(handled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) struct iommu_sva *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) struct iommu_sva *sva = ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) struct intel_svm_dev *sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) unsigned int flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * TODO: Consolidate with generic iommu-sva bind after it is merged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * It will require shared SVM data structures, i.e. combine io_mm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * and intel_svm etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) if (drvdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) flags = *(unsigned int *)drvdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) sva = ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) else if (sdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) sva = &sdev->sva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) WARN(!sdev, "SVM bind succeeded with no sdev!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) return sva;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) void intel_svm_unbind(struct iommu_sva *sva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) sdev = to_intel_svm_dev(sva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) intel_svm_unbind_mm(sdev->dev, sdev->pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) u32 intel_svm_get_pasid(struct iommu_sva *sva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) struct intel_svm_dev *sdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) u32 pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) sdev = to_intel_svm_dev(sva);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) pasid = sdev->pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) return pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) int intel_svm_page_response(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) struct iommu_fault_event *evt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) struct iommu_page_response *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) struct iommu_fault_page_request *prm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) struct intel_svm_dev *sdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) struct intel_svm *svm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) struct intel_iommu *iommu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) bool private_present;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) bool pasid_present;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) bool last_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) u8 bus, devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) u16 sid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) if (!dev || !dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) iommu = device_to_iommu(dev, &bus, &devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) if (!iommu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (!msg || !evt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) mutex_lock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) prm = &evt->fault.prm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) sid = PCI_DEVID(bus, devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (!pasid_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) if (prm->pasid == 0 || prm->pasid >= PASID_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (ret || !sdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * For responses from userspace, need to make sure that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) * pasid has been bound to its mm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) if (svm->flags & SVM_FLAG_GUEST_MODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) mm = get_task_mm(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (!mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (mm != svm->mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * Per VT-d spec. v3.0 ch7.7, system software must respond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * with page group response if private data is present (PDP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * or last page in group (LPIG) bit is set. This is an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * additional VT-d requirement beyond PCI ATS spec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) if (last_page || private_present) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) struct qi_desc desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) QI_PGRP_PASID_P(pasid_present) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) QI_PGRP_PDP(private_present) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) QI_PGRP_RESP_CODE(msg->code) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) QI_PGRP_RESP_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) desc.qw2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) desc.qw3 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (private_present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) memcpy(&desc.qw2, prm->private_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) sizeof(prm->private_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) qi_submit_sync(iommu, &desc, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) mutex_unlock(&pasid_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags