^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * intel-pasid.c - PASID idr, table and entry manipulation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2018 Intel Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Lu Baolu <baolu.lu@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define pr_fmt(fmt) "DMAR: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/dmar.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/intel-iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/iommu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/pci-ats.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "pasid.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Intel IOMMU system wide PASID name space:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static DEFINE_SPINLOCK(pasid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) u32 intel_pasid_max_id = PASID_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) u8 status_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) u64 res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) raw_spin_lock_irqsave(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) dmar_writeq(iommu->reg + DMAR_VCMD_REG, VCMD_CMD_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) !(res & VCMD_VRSP_IP), res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) status_code = VCMD_VRSP_SC(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) switch (status_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) case VCMD_VRSP_SC_SUCCESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) *pasid = VCMD_VRSP_RESULT_PASID(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) case VCMD_VRSP_SC_NO_PASID_AVAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) pr_info("IOMMU: %s: No PASID available\n", iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) pr_warn("IOMMU: %s: Unexpected error code %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) iommu->name, status_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u8 status_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u64 res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) raw_spin_lock_irqsave(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) dmar_writeq(iommu->reg + DMAR_VCMD_REG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) VCMD_CMD_OPERAND(pasid) | VCMD_CMD_FREE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) IOMMU_WAIT_OP(iommu, DMAR_VCRSP_REG, dmar_readq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) !(res & VCMD_VRSP_IP), res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) status_code = VCMD_VRSP_SC(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) switch (status_code) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) case VCMD_VRSP_SC_SUCCESS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) case VCMD_VRSP_SC_INVALID_PASID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) pr_info("IOMMU: %s: Invalid PASID\n", iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pr_warn("IOMMU: %s: Unexpected error code %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) iommu->name, status_code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * Per device pasid table management:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) device_attach_pasid_table(struct device_domain_info *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct pasid_table *pasid_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) info->pasid_table = pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) list_add(&info->table, &pasid_table->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) device_detach_pasid_table(struct device_domain_info *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct pasid_table *pasid_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) info->pasid_table = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) list_del(&info->table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct pasid_table_opaque {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct pasid_table **pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) int segment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) int devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static int search_pasid_table(struct device_domain_info *info, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct pasid_table_opaque *data = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (info->iommu->segment == data->segment &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) info->bus == data->bus &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) info->devfn == data->devfn &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) info->pasid_table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) *data->pasid_table = info->pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static int get_alias_pasid_table(struct pci_dev *pdev, u16 alias, void *opaque)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct pasid_table_opaque *data = opaque;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) data->segment = pci_domain_nr(pdev->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) data->bus = PCI_BUS_NUM(alias);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) data->devfn = alias & 0xff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return for_each_device_domain(&search_pasid_table, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * Allocate a pasid table for @dev. It should be called in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * single-thread context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) int intel_pasid_alloc_table(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct pasid_table *pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct pasid_table_opaque data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct page *pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) u32 max_pasid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) int ret, order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* DMA alias device already has a pasid table, use it: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) data.pasid_table = &pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) ret = pci_for_each_dma_alias(to_pci_dev(dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) &get_alias_pasid_table, &data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) goto attach_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (!pasid_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) INIT_LIST_HEAD(&pasid_table->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (info->pasid_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) max_pasid = min_t(u32, pci_max_pasids(to_pci_dev(dev)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) intel_pasid_max_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) size = max_pasid >> (PASID_PDE_SHIFT - 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) order = size ? get_order(size) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) pages = alloc_pages_node(info->iommu->node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) GFP_KERNEL | __GFP_ZERO, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (!pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) kfree(pasid_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) pasid_table->table = page_address(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) pasid_table->order = order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) attach_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) device_attach_pasid_table(info, pasid_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) void intel_pasid_free_table(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct pasid_table *pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct pasid_dir_entry *dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct pasid_entry *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int i, max_pde;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (!info || !dev_is_pci(dev) || !info->pasid_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) pasid_table = info->pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) device_detach_pasid_table(info, pasid_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (!list_empty(&pasid_table->dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* Free scalable mode PASID directory tables: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) dir = pasid_table->table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) for (i = 0; i < max_pde; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) table = get_pasid_table_from_pde(&dir[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) free_pgtable_page(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) free_pages((unsigned long)pasid_table->table, pasid_table->order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) kfree(pasid_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct pasid_table *intel_pasid_get_table(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return info->pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) int intel_pasid_get_dev_max_id(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (!info || !info->pasid_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) return info->pasid_table->max_pasid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct pasid_table *pasid_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct pasid_dir_entry *dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct pasid_entry *entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) int dir_index, index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) pasid_table = intel_pasid_get_table(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (WARN_ON(!pasid_table || pasid >= intel_pasid_get_dev_max_id(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) dir = pasid_table->table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) dir_index = pasid >> PASID_PDE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) index = pasid & PASID_PTE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) spin_lock(&pasid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) entries = get_pasid_table_from_pde(&dir[dir_index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (!entries) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) entries = alloc_pgtable_page(info->iommu->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) if (!entries) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) spin_unlock(&pasid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) WRITE_ONCE(dir[dir_index].val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) (u64)virt_to_phys(entries) | PASID_PTE_PRESENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) spin_unlock(&pasid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return &entries[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * Interfaces for PASID table entry manipulation:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static inline void pasid_clear_entry(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) WRITE_ONCE(pe->val[0], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) WRITE_ONCE(pe->val[1], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) WRITE_ONCE(pe->val[2], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) WRITE_ONCE(pe->val[3], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) WRITE_ONCE(pe->val[4], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) WRITE_ONCE(pe->val[5], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) WRITE_ONCE(pe->val[6], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) WRITE_ONCE(pe->val[7], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static inline void pasid_clear_entry_with_fpd(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) WRITE_ONCE(pe->val[0], PASID_PTE_FPD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) WRITE_ONCE(pe->val[1], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) WRITE_ONCE(pe->val[2], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) WRITE_ONCE(pe->val[3], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) WRITE_ONCE(pe->val[4], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) WRITE_ONCE(pe->val[5], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) WRITE_ONCE(pe->val[6], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) WRITE_ONCE(pe->val[7], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) intel_pasid_clear_entry(struct device *dev, u32 pasid, bool fault_ignore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct pasid_entry *pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) pe = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (WARN_ON(!pe))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (fault_ignore && pasid_pte_is_present(pe))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pasid_clear_entry_with_fpd(pe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) pasid_clear_entry(pe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) u64 old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) old = READ_ONCE(*ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) WRITE_ONCE(*ptr, (old & ~mask) | bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) pasid_set_domain_id(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * Get domain ID value of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static inline u16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) pasid_get_domain_id(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return (u16)(READ_ONCE(pe->val[1]) & GENMASK_ULL(15, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) pasid_set_slptr(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) pasid_set_address_width(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) pasid_set_translation_type(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * Enable fault processing by clearing the FPD(Fault Processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * Disable) field (Bit 1) of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static inline void pasid_set_fault_enable(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) pasid_set_bits(&pe->val[0], 1 << 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) static inline void pasid_set_sre(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) pasid_set_bits(&pe->val[2], 1 << 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * Setup the P(Present) field (Bit 0) of a scalable mode PASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static inline void pasid_set_present(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pasid_set_bits(&pe->val[0], 1 << 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) pasid_set_bits(&pe->val[1], 1 << 23, value << 23);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * Setup the Page Snoop (PGSNP) field (Bit 88) of a scalable mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) pasid_set_pgsnp(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) pasid_set_bits(&pe->val[1], 1ULL << 24, 1ULL << 24);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Setup the First Level Page table Pointer field (Bit 140~191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) pasid_set_flptr(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) pasid_set_bits(&pe->val[2], VTD_PAGE_MASK, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * Setup the First Level Paging Mode field (Bit 130~131) of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) pasid_set_flpm(struct pasid_entry *pe, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * Setup the Extended Access Flag Enable (EAFE) field (Bit 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * of a scalable mode PASID entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) pasid_set_eafe(struct pasid_entry *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) pasid_set_bits(&pe->val[2], 1 << 7, 1 << 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) u16 did, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) struct qi_desc desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) desc.qw0 = QI_PC_DID(did) | QI_PC_GRAN(QI_PC_PASID_SEL) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) QI_PC_PASID(pasid) | QI_PC_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) desc.qw1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) desc.qw2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) desc.qw3 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) qi_submit_sync(iommu, &desc, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) struct device_domain_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) u16 sid, qdep, pfsid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) info = get_domain_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (!info || !info->ats_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) sid = info->bus << 8 | info->devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) qdep = info->ats_qdep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) pfsid = info->pfsid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * When PASID 0 is used, it indicates RID2PASID(DMA request w/o PASID),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * devTLB flush w/o PASID should be used. For non-zero PASID under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * SVA usage, device could do DMA with multiple PASIDs. It is more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * efficient to flush devTLB specific to the PASID.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (pasid == PASID_RID2PASID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) qi_flush_dev_iotlb_pasid(iommu, sid, pfsid, pasid, qdep, 0, 64 - VTD_PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) u32 pasid, bool fault_ignore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) struct pasid_entry *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) u16 did, pgtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) pte = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (WARN_ON(!pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) did = pasid_get_domain_id(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) pgtt = pasid_pte_get_pgtt(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) intel_pasid_clear_entry(dev, pasid, fault_ignore);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (!ecap_coherent(iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) clflush_cache_range(pte, sizeof(*pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) pasid_cache_invalidation_with_pasid(iommu, did, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (pgtt == PASID_ENTRY_PGTT_PT || pgtt == PASID_ENTRY_PGTT_FL_ONLY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) /* Device IOTLB doesn't need to be flushed in caching mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (!cap_caching_mode(iommu->cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) devtlb_invalidation_with_pasid(iommu, dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static void pasid_flush_caches(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) struct pasid_entry *pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) u32 pasid, u16 did)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (!ecap_coherent(iommu->ecap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) clflush_cache_range(pte, sizeof(*pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (cap_caching_mode(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) pasid_cache_invalidation_with_pasid(iommu, did, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) qi_flush_piotlb(iommu, did, pasid, 0, -1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) iommu_flush_write_buffer(iommu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * Set up the scalable mode pasid table entry for first only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * translation type.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) int intel_pasid_setup_first_level(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct device *dev, pgd_t *pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) u32 pasid, u16 did, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) struct pasid_entry *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if (!ecap_flts(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) pr_err("No first level translation support on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) pte = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (WARN_ON(!pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) pasid_clear_entry(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* Setup the first level page table pointer: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) pasid_set_flptr(pte, (u64)__pa(pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (flags & PASID_FLAG_SUPERVISOR_MODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (!ecap_srs(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) pr_err("No supervisor request support on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) pasid_set_sre(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) if (flags & PASID_FLAG_FL5LP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (cap_5lp_support(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) pasid_set_flpm(pte, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) pr_err("No 5-level paging support for first-level\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) pasid_clear_entry(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (flags & PASID_FLAG_PAGE_SNOOP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) pasid_set_pgsnp(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) pasid_set_domain_id(pte, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) pasid_set_address_width(pte, iommu->agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* Setup Present and PASID Granular Transfer Type: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) pasid_set_translation_type(pte, PASID_ENTRY_PGTT_FL_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) pasid_set_present(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) pasid_flush_caches(iommu, pte, pasid, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * Skip top levels of page tables for iommu which has less agaw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * than default. Unnecessary for PT mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static inline int iommu_skip_agaw(struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct dma_pte **pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) int agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) *pgd = phys_to_virt(dma_pte_addr(*pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (!dma_pte_present(*pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * Set up the scalable mode pasid entry for second only translation type.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) int intel_pasid_setup_second_level(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct pasid_entry *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) struct dma_pte *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) u64 pgd_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) int agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) u16 did;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * If hardware advertises no support for second level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * translation, return directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (!ecap_slts(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) pr_err("No second level translation support on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) pgd = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) agaw = iommu_skip_agaw(domain, iommu, &pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (agaw < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) dev_err(dev, "Invalid domain page table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) pgd_val = virt_to_phys(pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) pte = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (!pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) pasid_clear_entry(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) pasid_set_domain_id(pte, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) pasid_set_slptr(pte, pgd_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) pasid_set_address_width(pte, agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) pasid_set_translation_type(pte, PASID_ENTRY_PGTT_SL_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) pasid_set_fault_enable(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (domain->domain.type == IOMMU_DOMAIN_UNMANAGED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) pasid_set_pgsnp(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * Since it is a second level only translation setup, we should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * set SRE bit as well (addresses are expected to be GPAs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (pasid != PASID_RID2PASID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) pasid_set_sre(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) pasid_set_present(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) pasid_flush_caches(iommu, pte, pasid, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) * Set up the scalable mode pasid entry for passthrough translation type.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) struct dmar_domain *domain,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) struct device *dev, u32 pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) u16 did = FLPT_DEFAULT_DID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) struct pasid_entry *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) pte = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) if (!pte) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) pasid_clear_entry(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) pasid_set_domain_id(pte, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) pasid_set_address_width(pte, iommu->agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) pasid_set_translation_type(pte, PASID_ENTRY_PGTT_PT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) pasid_set_fault_enable(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * We should set SRE bit as well since the addresses are expected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) * to be GPAs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) pasid_set_sre(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) pasid_set_present(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) pasid_flush_caches(iommu, pte, pasid, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) intel_pasid_setup_bind_data(struct intel_iommu *iommu, struct pasid_entry *pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct iommu_gpasid_bind_data_vtd *pasid_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * Not all guest PASID table entry fields are passed down during bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * here we only set up the ones that are dependent on guest settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * Execution related bits such as NXE, SMEP are not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * Other fields, such as snoop related, are set based on host needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) * regardless of guest settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_SRE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (!ecap_srs(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) pr_err_ratelimited("No supervisor request support on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) pasid_set_sre(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_EAFE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (!ecap_eafs(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) pr_err_ratelimited("No extended access flag support on %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) pasid_set_eafe(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * Memory type is only applicable to devices inside processor coherent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * domain. Will add MTS support once coherent devices are available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (pasid_data->flags & IOMMU_SVA_VTD_GPASID_MTS_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) pr_warn_ratelimited("No memory type support %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * intel_pasid_setup_nested() - Set up PASID entry for nested translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * This could be used for guest shared virtual address. In this case, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * first level page tables are used for GVA-GPA translation in the guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * second level page tables are used for GPA-HPA translation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * @iommu: IOMMU which the device belong to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * @dev: Device to be set up for translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * @gpgd: FLPTPTR: First Level Page translation pointer in GPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * @pasid: PASID to be programmed in the device PASID table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * @pasid_data: Additional PASID info from the guest bind request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * @domain: Domain info for setting up second level page tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * @addr_width: Address width of the first level (guest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) pgd_t *gpgd, u32 pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) struct iommu_gpasid_bind_data_vtd *pasid_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) struct dmar_domain *domain, int addr_width)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) struct pasid_entry *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) struct dma_pte *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) u64 pgd_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) int agaw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) u16 did;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) if (!ecap_nest(iommu->ecap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) pr_err_ratelimited("IOMMU: %s: No nested translation support\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) iommu->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (!(domain->flags & DOMAIN_FLAG_NESTING_MODE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) pr_err_ratelimited("Domain is not in nesting mode, %x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) domain->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) pte = intel_pasid_get_entry(dev, pasid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) if (WARN_ON(!pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * Caller must ensure PASID entry is not in use, i.e. not bind the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * same PASID to the same device twice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (pasid_pte_is_present(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) pasid_clear_entry(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) /* Sanity checking performed by caller to make sure address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * width matching in two dimensions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * 1. CPU vs. IOMMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * 2. Guest vs. Host.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) switch (addr_width) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) case ADDR_WIDTH_5LEVEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (!cpu_feature_enabled(X86_FEATURE_LA57) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) !cap_5lp_support(iommu->cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) dev_err_ratelimited(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) "5-level paging not supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) pasid_set_flpm(pte, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) case ADDR_WIDTH_4LEVEL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) pasid_set_flpm(pte, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) dev_err_ratelimited(dev, "Invalid guest address width %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) addr_width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) /* First level PGD is in GPA, must be supported by the second level */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if ((uintptr_t)gpgd > domain->max_addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) dev_err_ratelimited(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) "Guest PGD %lx not supported, max %llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) (uintptr_t)gpgd, domain->max_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) pasid_set_flptr(pte, (uintptr_t)gpgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) ret = intel_pasid_setup_bind_data(iommu, pte, pasid_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /* Setup the second level based on the given domain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) pgd = domain->pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) agaw = iommu_skip_agaw(domain, iommu, &pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) if (agaw < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) dev_err_ratelimited(dev, "Invalid domain page table\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) pgd_val = virt_to_phys(pgd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) pasid_set_slptr(pte, pgd_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) pasid_set_fault_enable(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) did = domain->iommu_did[iommu->seq_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) pasid_set_domain_id(pte, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) pasid_set_address_width(pte, agaw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) pasid_set_translation_type(pte, PASID_ENTRY_PGTT_NESTED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) pasid_set_present(pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) pasid_flush_caches(iommu, pte, pasid, did);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags