^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * PCI Peer 2 Peer DMA support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2016-2018, Logan Gunthorpe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2016-2017, Microsemi Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (c) 2017, Christoph Hellwig
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (c) 2018, Eideticom Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define pr_fmt(fmt) "pci-p2pdma: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/ctype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/pci-p2pdma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/genalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/memremap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/percpu-refcount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/seq_buf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/xarray.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) enum pci_p2pdma_map_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) PCI_P2PDMA_MAP_UNKNOWN = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) PCI_P2PDMA_MAP_NOT_SUPPORTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) PCI_P2PDMA_MAP_BUS_ADDR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) PCI_P2PDMA_MAP_THRU_HOST_BRIDGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct pci_p2pdma {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct gen_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) bool p2pmem_published;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct xarray map_types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct pci_p2pdma_pagemap {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct dev_pagemap pgmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct pci_dev *provider;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) u64 bus_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static struct pci_p2pdma_pagemap *to_p2p_pgmap(struct dev_pagemap *pgmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return container_of(pgmap, struct pci_p2pdma_pagemap, pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static ssize_t size_show(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) size_t size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) if (pdev->p2pdma->pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) size = gen_pool_size(pdev->p2pdma->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return scnprintf(buf, PAGE_SIZE, "%zd\n", size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static DEVICE_ATTR_RO(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static ssize_t available_show(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) size_t avail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (pdev->p2pdma->pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) avail = gen_pool_avail(pdev->p2pdma->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return scnprintf(buf, PAGE_SIZE, "%zd\n", avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static DEVICE_ATTR_RO(available);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static ssize_t published_show(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return scnprintf(buf, PAGE_SIZE, "%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) pdev->p2pdma->p2pmem_published);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static DEVICE_ATTR_RO(published);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static struct attribute *p2pmem_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) &dev_attr_size.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) &dev_attr_available.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) &dev_attr_published.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static const struct attribute_group p2pmem_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .attrs = p2pmem_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) .name = "p2pmem",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static void pci_p2pdma_release(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct pci_dev *pdev = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct pci_p2pdma *p2pdma = pdev->p2pdma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (!p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /* Flush and disable pci_alloc_p2p_mem() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pdev->p2pdma = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) gen_pool_destroy(p2pdma->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) xa_destroy(&p2pdma->map_types);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static int pci_p2pdma_setup(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int error = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct pci_p2pdma *p2p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (!p2p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) xa_init(&p2p->map_types);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (!p2p->pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) goto out_pool_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) pdev->p2pdma = p2p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) goto out_pool_destroy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) out_pool_destroy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) pdev->p2pdma = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) gen_pool_destroy(p2p->pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) devm_kfree(&pdev->dev, p2p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * pci_p2pdma_add_resource - add memory for use as p2p memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * @pdev: the device to add the memory to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * @bar: PCI BAR to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * @size: size of the memory to add, may be zero to use the whole BAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * @offset: offset into the PCI BAR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * The memory will be given ZONE_DEVICE struct pages so that it may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * be used with any DMA request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct pci_p2pdma_pagemap *p2p_pgmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct dev_pagemap *pgmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (offset >= pci_resource_len(pdev, bar))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (!size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) size = pci_resource_len(pdev, bar) - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (size + offset > pci_resource_len(pdev, bar))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!pdev->p2pdma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) error = pci_p2pdma_setup(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (!p2p_pgmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) pgmap = &p2p_pgmap->pgmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) pgmap->range.start = pci_resource_start(pdev, bar) + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) pgmap->range.end = pgmap->range.start + size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) pgmap->nr_range = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) p2p_pgmap->provider = pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) p2p_pgmap->bus_offset = pci_bus_address(pdev, bar) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) pci_resource_start(pdev, bar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) addr = devm_memremap_pages(&pdev->dev, pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (IS_ERR(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) error = PTR_ERR(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) goto pgmap_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) pci_bus_address(pdev, bar) + offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) range_len(&pgmap->range), dev_to_node(&pdev->dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) pgmap->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) goto pages_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) pgmap->range.start, pgmap->range.end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) pages_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) devm_memunmap_pages(&pdev->dev, pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) pgmap_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) devm_kfree(&pdev->dev, pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * Note this function returns the parent PCI device with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * reference taken. It is the caller's responsibility to drop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * the reference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static struct pci_dev *find_parent_pci_dev(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct device *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) dev = get_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) while (dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (dev_is_pci(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) parent = get_device(dev->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) put_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) dev = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * Check if a PCI bridge has its ACS redirection bits set to redirect P2P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * TLPs upstream via ACS. Returns 1 if the packets will be redirected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * upstream, 0 otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static int pci_bridge_has_acs_redir(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) int pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) u16 ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) pos = pdev->acs_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (!pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (ctrl & (PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) seq_buf_printf(buf, "%s;", pci_name(pdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static bool cpu_supports_p2pdma(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct cpuinfo_x86 *c = &cpu_data(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* Any AMD CPU whose family ID is Zen or newer supports p2pdma */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static const struct pci_p2pdma_whitelist_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) unsigned short vendor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) unsigned short device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) REQ_SAME_HOST_BRIDGE = 1 << 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) } flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) } pci_p2pdma_whitelist[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* Intel Xeon E5/Core i7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {PCI_VENDOR_ID_INTEL, 0x3c00, REQ_SAME_HOST_BRIDGE},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {PCI_VENDOR_ID_INTEL, 0x3c01, REQ_SAME_HOST_BRIDGE},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /* Intel Xeon E7 v3/Xeon E5 v3/Core i7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {PCI_VENDOR_ID_INTEL, 0x2f00, REQ_SAME_HOST_BRIDGE},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {PCI_VENDOR_ID_INTEL, 0x2f01, REQ_SAME_HOST_BRIDGE},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /* Intel SkyLake-E */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {PCI_VENDOR_ID_INTEL, 0x2030, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {PCI_VENDOR_ID_INTEL, 0x2031, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {PCI_VENDOR_ID_INTEL, 0x2032, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {PCI_VENDOR_ID_INTEL, 0x2033, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {PCI_VENDOR_ID_INTEL, 0x2020, 0},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * This lookup function tries to find the PCI device corresponding to a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * host bridge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * It assumes the host bridge device is the first PCI device in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * bus->devices list and that the devfn is 00.0. These assumptions should hold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * for all the devices in the whitelist above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * This function is equivalent to pci_get_slot(host->bus, 0), however it does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * not take the pci_bus_sem lock seeing __host_bridge_whitelist() must not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * For this to be safe, the caller should hold a reference to a device on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * bridge, which should ensure the host_bridge device will not be freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * or removed from the head of the devices list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static struct pci_dev *pci_host_bridge_dev(struct pci_host_bridge *host)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) struct pci_dev *root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) root = list_first_entry_or_null(&host->bus->devices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) struct pci_dev, bus_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (!root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (root->devfn != PCI_DEVFN(0, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static bool __host_bridge_whitelist(struct pci_host_bridge *host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) bool same_host_bridge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct pci_dev *root = pci_host_bridge_dev(host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) const struct pci_p2pdma_whitelist_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) unsigned short vendor, device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (!root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) vendor = root->vendor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) device = root->device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (vendor != entry->vendor || device != entry->device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * If we can't find a common upstream bridge take a look at the root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * complex and compare it to a whitelist of known good hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (host_a == host_b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return __host_bridge_whitelist(host_a, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (__host_bridge_whitelist(host_a, false) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) __host_bridge_whitelist(host_b, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) static enum pci_p2pdma_map_type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) __upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) int *dist, bool *acs_redirects, struct seq_buf *acs_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) struct pci_dev *a = provider, *b = client, *bb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int dist_a = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) int dist_b = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) int acs_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (acs_redirects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) *acs_redirects = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * Note, we don't need to take references to devices returned by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * pci_upstream_bridge() seeing we hold a reference to a child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * device which will already hold a reference to the upstream bridge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) while (a) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) dist_b = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (pci_bridge_has_acs_redir(a)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) seq_buf_print_bus_devfn(acs_list, a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) acs_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) bb = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) while (bb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (a == bb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) goto check_b_path_acs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) bb = pci_upstream_bridge(bb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) dist_b++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) a = pci_upstream_bridge(a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) dist_a++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (dist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) *dist = dist_a + dist_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) check_b_path_acs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) bb = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) while (bb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (a == bb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (pci_bridge_has_acs_redir(bb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) seq_buf_print_bus_devfn(acs_list, bb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) acs_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) bb = pci_upstream_bridge(bb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (dist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) *dist = dist_a + dist_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (acs_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (acs_redirects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) *acs_redirects = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) return PCI_P2PDMA_MAP_BUS_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) static unsigned long map_types_idx(struct pci_dev *client)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return (pci_domain_nr(client->bus) << 16) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) (client->bus->number << 8) | client->devfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * Find the distance through the nearest common upstream bridge between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * two PCI devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * If the two devices are the same device then 0 will be returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * If there are two virtual functions of the same device behind the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * bridge port then 2 will be returned (one step down to the PCIe switch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * then one step back to the same device).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * In the case where two devices are connected to the same PCIe switch, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * value 4 will be returned. This corresponds to the following PCI tree:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * -+ Root Port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * \+ Switch Upstream Port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * +-+ Switch Downstream Port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * + \- Device A
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * \-+ Switch Downstream Port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * \- Device B
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * The distance is 4 because we traverse from Device A through the downstream
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * port of the switch, to the common upstream port, back up to the second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * downstream port and then to Device B.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * Any two devices that cannot communicate using p2pdma will return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * PCI_P2PDMA_MAP_NOT_SUPPORTED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * Any two devices that have a data path that goes through the host bridge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * will consult a whitelist. If the host bridges are on the whitelist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * this function will return PCI_P2PDMA_MAP_THRU_HOST_BRIDGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * If either bridge is not on the whitelist this function returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * PCI_P2PDMA_MAP_NOT_SUPPORTED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * If a bridge which has any ACS redirection bits set is in the path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * acs_redirects will be set to true. In this case, a list of all infringing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * bridge addresses will be populated in acs_list (assuming it's non-null)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * for printk purposes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static enum pci_p2pdma_map_type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) int *dist, bool *acs_redirects, struct seq_buf *acs_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) enum pci_p2pdma_map_type map_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) map_type = __upstream_bridge_distance(provider, client, dist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) acs_redirects, acs_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (!cpu_supports_p2pdma() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) !host_bridge_whitelist(provider, client))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (provider->p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) xa_store(&provider->p2pdma->map_types, map_types_idx(client),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) xa_mk_value(map_type), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return map_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static enum pci_p2pdma_map_type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) upstream_bridge_distance_warn(struct pci_dev *provider, struct pci_dev *client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) int *dist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct seq_buf acs_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) bool acs_redirects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) seq_buf_init(&acs_list, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (!acs_list.buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ret = upstream_bridge_distance(provider, client, dist, &acs_redirects,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) &acs_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (acs_redirects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) pci_warn(client, "ACS redirect is set between the client and provider (%s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) pci_name(provider));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /* Drop final semicolon */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) acs_list.buffer[acs_list.len-1] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) pci_warn(client, "to disable ACS redirect for this path, add the kernel parameter: pci=disable_acs_redir=%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) acs_list.buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share an upstream bridge or whitelisted host bridge\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) pci_name(provider));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) kfree(acs_list.buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * pci_p2pdma_distance_many - Determine the cumulative distance between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * a p2pdma provider and the clients in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * @provider: p2pdma provider to check against the client list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * @clients: array of devices to check (NULL-terminated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * @num_clients: number of clients in the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * @verbose: if true, print warnings for devices when we return -1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * Returns -1 if any of the clients are not compatible, otherwise returns a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * positive number where a lower number is the preferable choice. (If there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * one client that's the same as the provider it will return 0, which is best
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * choice).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * "compatible" means the provider and the clients are either all behind
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * the same PCI root port or the host bridges connected to each of the devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * are listed in the 'pci_p2pdma_whitelist'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) int num_clients, bool verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) bool not_supported = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct pci_dev *pci_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int total_dist = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (num_clients == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) for (i = 0; i < num_clients; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) #ifdef CONFIG_DMA_VIRT_OPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (clients[i]->dma_ops == &dma_virt_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) dev_warn(clients[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) "cannot be used for peer-to-peer DMA because the driver makes use of dma_virt_ops\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) pci_client = find_parent_pci_dev(clients[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (!pci_client) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) dev_warn(clients[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) "cannot be used for peer-to-peer DMA as it is not a PCI device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) ret = upstream_bridge_distance_warn(provider,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) pci_client, &distance);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) ret = upstream_bridge_distance(provider, pci_client,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) &distance, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) pci_dev_put(pci_client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (ret == PCI_P2PDMA_MAP_NOT_SUPPORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) not_supported = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (not_supported && !verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) total_dist += distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (not_supported)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) return total_dist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) EXPORT_SYMBOL_GPL(pci_p2pdma_distance_many);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * pci_has_p2pmem - check if a given PCI device has published any p2pmem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * @pdev: PCI device to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) bool pci_has_p2pmem(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return pdev->p2pdma && pdev->p2pdma->p2pmem_published;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) EXPORT_SYMBOL_GPL(pci_has_p2pmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * pci_p2pmem_find - find a peer-to-peer DMA memory device compatible with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * the specified list of clients and shortest distance (as determined
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) * by pci_p2pmem_dma())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * @clients: array of devices to check (NULL-terminated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * @num_clients: number of client devices in the list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * If multiple devices are behind the same switch, the one "closest" to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) * client devices in use will be chosen first. (So if one of the providers is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * the same as one of the clients, that provider will be used ahead of any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * other providers that are unrelated). If multiple providers are an equal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * distance away, one will be chosen at random.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * Returns a pointer to the PCI device with a reference taken (use pci_dev_put
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * to return the reference) or NULL if no compatible device is found. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * found provider will also be assigned to the client list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) struct pci_dev *pdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) int distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) int closest_distance = INT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) struct pci_dev **closest_pdevs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) int dev_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) const int max_devs = PAGE_SIZE / sizeof(*closest_pdevs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) closest_pdevs = kmalloc(PAGE_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (!closest_pdevs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) while ((pdev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (!pci_has_p2pmem(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) distance = pci_p2pdma_distance_many(pdev, clients,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) num_clients, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (distance < 0 || distance > closest_distance)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (distance == closest_distance && dev_cnt >= max_devs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (distance < closest_distance) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) for (i = 0; i < dev_cnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) pci_dev_put(closest_pdevs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) dev_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) closest_distance = distance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) closest_pdevs[dev_cnt++] = pci_dev_get(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (dev_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) pdev = pci_dev_get(closest_pdevs[prandom_u32_max(dev_cnt)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) for (i = 0; i < dev_cnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) pci_dev_put(closest_pdevs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) kfree(closest_pdevs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) return pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) EXPORT_SYMBOL_GPL(pci_p2pmem_find_many);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * pci_alloc_p2p_mem - allocate peer-to-peer DMA memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * @pdev: the device to allocate memory from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * @size: number of bytes to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * Returns the allocated memory or NULL on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) void *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct percpu_ref *ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * Pairs with synchronize_rcu() in pci_p2pdma_release() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * ensure pdev->p2pdma is non-NULL for the duration of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) * read-lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (unlikely(!pdev->p2pdma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) ret = (void *)gen_pool_alloc_owner(pdev->p2pdma->pool, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) (void **) &ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (unlikely(!percpu_ref_tryget_live(ref))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) gen_pool_free(pdev->p2pdma->pool, (unsigned long) ret, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * pci_free_p2pmem - free peer-to-peer DMA memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * @pdev: the device the memory was allocated from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * @addr: address of the memory that was allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * @size: number of bytes that were allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct percpu_ref *ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) gen_pool_free_owner(pdev->p2pdma->pool, (uintptr_t)addr, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) (void **) &ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) percpu_ref_put(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) EXPORT_SYMBOL_GPL(pci_free_p2pmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * pci_virt_to_bus - return the PCI bus address for a given virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * address obtained with pci_alloc_p2pmem()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * @pdev: the device the memory was allocated from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * @addr: address of the memory that was allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) pci_bus_addr_t pci_p2pmem_virt_to_bus(struct pci_dev *pdev, void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (!pdev->p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * Note: when we added the memory to the pool we used the PCI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * bus address as the physical address. So gen_pool_virt_to_phys()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * actually returns the bus address despite the misleading name.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) return gen_pool_virt_to_phys(pdev->p2pdma->pool, (unsigned long)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) EXPORT_SYMBOL_GPL(pci_p2pmem_virt_to_bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * @pdev: the device to allocate memory from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * @nents: the number of SG entries in the list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * @length: number of bytes to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * Return: %NULL on error or &struct scatterlist pointer and @nents on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct scatterlist *pci_p2pmem_alloc_sgl(struct pci_dev *pdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) unsigned int *nents, u32 length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) sg = kmalloc(sizeof(*sg), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) sg_init_table(sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) addr = pci_alloc_p2pmem(pdev, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) goto out_free_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) sg_set_buf(sg, addr, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) *nents = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) out_free_sg:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) kfree(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) EXPORT_SYMBOL_GPL(pci_p2pmem_alloc_sgl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * @pdev: the device to allocate memory from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * @sgl: the allocated scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) void pci_p2pmem_free_sgl(struct pci_dev *pdev, struct scatterlist *sgl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) for_each_sg(sgl, sg, INT_MAX, count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) pci_free_p2pmem(pdev, sg_virt(sg), sg->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) kfree(sgl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) EXPORT_SYMBOL_GPL(pci_p2pmem_free_sgl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * other devices with pci_p2pmem_find()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * @pdev: the device with peer-to-peer DMA memory to publish
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * @publish: set to true to publish the memory, false to unpublish it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * Published memory can be used by other PCI device drivers for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * peer-2-peer DMA operations. Non-published memory is reserved for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * exclusive use of the device driver that registers the peer-to-peer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) void pci_p2pmem_publish(struct pci_dev *pdev, bool publish)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (pdev->p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) pdev->p2pdma->p2pmem_published = publish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) EXPORT_SYMBOL_GPL(pci_p2pmem_publish);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct pci_dev *provider,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) struct pci_dev *client)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (!provider->p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return PCI_P2PDMA_MAP_NOT_SUPPORTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return xa_to_value(xa_load(&provider->p2pdma->map_types,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) map_types_idx(client)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) static int __pci_p2pdma_map_sg(struct pci_p2pdma_pagemap *p2p_pgmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) struct device *dev, struct scatterlist *sg, int nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct scatterlist *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) phys_addr_t paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * p2pdma mappings are not compatible with devices that use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * dma_virt_ops. If the upper layers do the right thing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * this should never happen because it will be prevented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * by the check in pci_p2pdma_distance_many()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) #ifdef CONFIG_DMA_VIRT_OPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (WARN_ON_ONCE(dev->dma_ops == &dma_virt_ops))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) for_each_sg(sg, s, nents, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) paddr = sg_phys(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) s->dma_address = paddr - p2p_pgmap->bus_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) sg_dma_len(s) = s->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) return nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * pci_p2pdma_map_sg - map a PCI peer-to-peer scatterlist for DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * @dev: device doing the DMA request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * @sg: scatter list to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * @nents: elements in the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * @dir: DMA direction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * @attrs: DMA attributes passed to dma_map_sg() (if called)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * Scatterlists mapped with this function should be unmapped using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * pci_p2pdma_unmap_sg_attrs().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * Returns the number of SG entries mapped or 0 on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) int pci_p2pdma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) int nents, enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) struct pci_p2pdma_pagemap *p2p_pgmap =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) to_p2p_pgmap(sg_page(sg)->pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) struct pci_dev *client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (WARN_ON_ONCE(!dev_is_pci(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) client = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) switch (pci_p2pdma_map_type(p2p_pgmap->provider, client)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) return dma_map_sg_attrs(dev, sg, nents, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) case PCI_P2PDMA_MAP_BUS_ADDR:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) return __pci_p2pdma_map_sg(p2p_pgmap, dev, sg, nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) EXPORT_SYMBOL_GPL(pci_p2pdma_map_sg_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * pci_p2pdma_unmap_sg - unmap a PCI peer-to-peer scatterlist that was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * mapped with pci_p2pdma_map_sg()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * @dev: device doing the DMA request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * @sg: scatter list to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * @nents: number of elements returned by pci_p2pdma_map_sg()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * @dir: DMA direction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * @attrs: DMA attributes passed to dma_unmap_sg() (if called)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) void pci_p2pdma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) int nents, enum dma_data_direction dir, unsigned long attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) struct pci_p2pdma_pagemap *p2p_pgmap =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) to_p2p_pgmap(sg_page(sg)->pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) enum pci_p2pdma_map_type map_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) struct pci_dev *client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (WARN_ON_ONCE(!dev_is_pci(dev)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) client = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) map_type = pci_p2pdma_map_type(p2p_pgmap->provider, client);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) dma_unmap_sg_attrs(dev, sg, nents, dir, attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) EXPORT_SYMBOL_GPL(pci_p2pdma_unmap_sg_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * to enable p2pdma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * @page: contents of the value to be stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * @p2p_dev: returns the PCI device that was selected to be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * (if one was specified in the stored value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) * @use_p2pdma: returns whether to enable p2pdma or not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) * Parses an attribute value to decide whether to enable p2pdma.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) * The value can select a PCI device (using its full BDF device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * name) or a boolean (in any format strtobool() accepts). A false
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) * value disables p2pdma, a true value expects the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) * to automatically find a compatible device and specifying a PCI device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * expects the caller to use the specific provider.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) * pci_p2pdma_enable_show() should be used as the show operation for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) * the attribute.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) bool *use_p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) dev = bus_find_device_by_name(&pci_bus_type, NULL, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) if (dev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) *use_p2pdma = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) *p2p_dev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (!pci_has_p2pmem(*p2p_dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) pci_err(*p2p_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) "PCI device has no peer-to-peer memory: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) pci_dev_put(*p2p_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) } else if ((page[0] == '0' || page[0] == '1') && !iscntrl(page[1])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * If the user enters a PCI device that doesn't exist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * like "0000:01:00.1", we don't want strtobool to think
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * it's a '0' when it's clearly not what the user wanted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * So we require 0's and 1's to be exactly one character.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) } else if (!strtobool(page, use_p2pdma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) pr_err("No such PCI device: %.*s\n", (int)strcspn(page, "\n"), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) EXPORT_SYMBOL_GPL(pci_p2pdma_enable_store);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * whether p2pdma is enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * @page: contents of the stored value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) * @p2p_dev: the selected p2p device (NULL if no device is selected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * @use_p2pdma: whether p2pdma has been enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * Attributes that use pci_p2pdma_enable_store() should use this function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * to show the value of the attribute.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * Returns 0 on success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) bool use_p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (!use_p2pdma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return sprintf(page, "0\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) if (!p2p_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) return sprintf(page, "1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return sprintf(page, "%s\n", pci_name(p2p_dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) EXPORT_SYMBOL_GPL(pci_p2pdma_enable_show);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags