Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Driver for FPGA Accelerated Function Unit (AFU) DMA Region Management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2017-2018 Intel Corporation, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Authors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *   Wu Hao <hao.wu@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include "dfl-afu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	afu->dma_regions = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * afu_dma_pin_pages - pin pages of given dma memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * @region: dma memory region to be pinned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  * Pin all the pages of given dfl_afu_dma_region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  * Return 0 for success or negative error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 			     struct dfl_afu_dma_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	int npages = region->length >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	struct device *dev = &pdata->dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	int ret, pinned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	ret = account_locked_vm(current->mm, npages, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	region->pages = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	if (!region->pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 		ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 		goto unlock_vm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	pinned = pin_user_pages_fast(region->user_addr, npages, FOLL_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 				     region->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	if (pinned < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		ret = pinned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		goto free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	} else if (pinned != npages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 		ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		goto unpin_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	dev_dbg(dev, "%d pages pinned\n", pinned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) unpin_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	unpin_user_pages(region->pages, pinned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) free_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	kfree(region->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) unlock_vm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	account_locked_vm(current->mm, npages, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75)  * afu_dma_unpin_pages - unpin pages of given dma memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77)  * @region: dma memory region to be unpinned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79)  * Unpin all the pages of given dfl_afu_dma_region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80)  * Return 0 for success or negative error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 				struct dfl_afu_dma_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	long npages = region->length >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	struct device *dev = &pdata->dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	unpin_user_pages(region->pages, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	kfree(region->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	account_locked_vm(current->mm, npages, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	dev_dbg(dev, "%ld pages unpinned\n", npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96)  * afu_dma_check_continuous_pages - check if pages are continuous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97)  * @region: dma memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99)  * Return true if pages of given dma memory region have continuous physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)  * address, otherwise return false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static bool afu_dma_check_continuous_pages(struct dfl_afu_dma_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	int npages = region->length >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	for (i = 0; i < npages - 1; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		if (page_to_pfn(region->pages[i]) + 1 !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 				page_to_pfn(region->pages[i + 1]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * dma_region_check_iova - check if memory area is fully contained in the region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  * @region: dma memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * @iova: address of the dma memory area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * @size: size of the dma memory area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  * Compare the dma memory area defined by @iova and @size with given dma region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  * Return true if memory area is fully contained in the region, otherwise false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static bool dma_region_check_iova(struct dfl_afu_dma_region *region,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 				  u64 iova, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	if (!size && region->iova != iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	return (region->iova <= iova) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		(region->length + region->iova >= iova + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)  * afu_dma_region_add - add given dma region to rbtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)  * @region: dma region to be added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)  * Return 0 for success, -EEXIST if dma region has already been added.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)  * Needs to be called with pdata->lock heold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static int afu_dma_region_add(struct dfl_feature_platform_data *pdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 			      struct dfl_afu_dma_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	struct rb_node **new, *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	dev_dbg(&pdata->dev->dev, "add region (iova = %llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		(unsigned long long)region->iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	new = &afu->dma_regions.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	while (*new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		struct dfl_afu_dma_region *this;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		this = container_of(*new, struct dfl_afu_dma_region, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		parent = *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 		if (dma_region_check_iova(this, region->iova, region->length))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 			return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		if (region->iova < this->iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 			new = &((*new)->rb_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		else if (region->iova > this->iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 			new = &((*new)->rb_right);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 			return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	rb_link_node(&region->node, parent, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	rb_insert_color(&region->node, &afu->dma_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)  * afu_dma_region_remove - remove given dma region from rbtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)  * @region: dma region to be removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)  * Needs to be called with pdata->lock heold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static void afu_dma_region_remove(struct dfl_feature_platform_data *pdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 				  struct dfl_afu_dma_region *region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	struct dfl_afu *afu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		(unsigned long long)region->iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	afu = dfl_fpga_pdata_get_private(pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	rb_erase(&region->node, &afu->dma_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  * afu_dma_region_destroy - destroy all regions in rbtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)  * Needs to be called with pdata->lock heold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) void afu_dma_region_destroy(struct dfl_feature_platform_data *pdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	struct rb_node *node = rb_first(&afu->dma_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	struct dfl_afu_dma_region *region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		region = container_of(node, struct dfl_afu_dma_region, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 			(unsigned long long)region->iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		rb_erase(node, &afu->dma_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		if (region->iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 			dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 				       region->iova, region->length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 				       DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		if (region->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 			afu_dma_unpin_pages(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		node = rb_next(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		kfree(region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)  * afu_dma_region_find - find the dma region from rbtree based on iova and size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)  * @iova: address of the dma memory area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  * @size: size of the dma memory area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)  * It finds the dma region from the rbtree based on @iova and @size:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)  * - if @size == 0, it finds the dma region which starts from @iova
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)  * - otherwise, it finds the dma region which fully contains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)  *   [@iova, @iova+size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)  * If nothing is matched returns NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  * Needs to be called with pdata->lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) struct dfl_afu_dma_region *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) afu_dma_region_find(struct dfl_feature_platform_data *pdata, u64 iova, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	struct rb_node *node = afu->dma_regions.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	struct device *dev = &pdata->dev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		struct dfl_afu_dma_region *region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		region = container_of(node, struct dfl_afu_dma_region, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		if (dma_region_check_iova(region, iova, size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 			dev_dbg(dev, "find region (iova = %llx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 				(unsigned long long)region->iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 			return region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		if (iova < region->iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 			node = node->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		else if (iova > region->iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 			node = node->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 			/* the iova region is not fully covered. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	dev_dbg(dev, "region with iova %llx and size %llx is not found\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		(unsigned long long)iova, (unsigned long long)size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)  * afu_dma_region_find_iova - find the dma region from rbtree by iova
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)  * @iova: address of the dma region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)  * Needs to be called with pdata->lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static struct dfl_afu_dma_region *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) afu_dma_region_find_iova(struct dfl_feature_platform_data *pdata, u64 iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	return afu_dma_region_find(pdata, iova, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)  * afu_dma_map_region - map memory region for dma
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)  * @user_addr: address of the memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)  * @length: size of the memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)  * @iova: pointer of iova address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)  * Map memory region defined by @user_addr and @length, and return dma address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)  * of the memory region via @iova.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)  * Return 0 for success, otherwise error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) int afu_dma_map_region(struct dfl_feature_platform_data *pdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		       u64 user_addr, u64 length, u64 *iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	struct dfl_afu_dma_region *region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	 * Check Inputs, only accept page-aligned user memory region with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	 * valid length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	if (!PAGE_ALIGNED(user_addr) || !PAGE_ALIGNED(length) || !length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	/* Check overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	if (user_addr + length < user_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	region = kzalloc(sizeof(*region), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	if (!region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	region->user_addr = user_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	region->length = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	/* Pin the user memory region */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	ret = afu_dma_pin_pages(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 		dev_err(&pdata->dev->dev, "failed to pin memory region\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 		goto free_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	/* Only accept continuous pages, return error else */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	if (!afu_dma_check_continuous_pages(region)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 		dev_err(&pdata->dev->dev, "pages are not continuous\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		goto unpin_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	/* As pages are continuous then start to do DMA mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	region->iova = dma_map_page(dfl_fpga_pdata_to_parent(pdata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 				    region->pages[0], 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 				    region->length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 				    DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	if (dma_mapping_error(dfl_fpga_pdata_to_parent(pdata), region->iova)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 		dev_err(&pdata->dev->dev, "failed to map for dma\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 		goto unpin_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	*iova = region->iova;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	mutex_lock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	ret = afu_dma_region_add(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	mutex_unlock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		dev_err(&pdata->dev->dev, "failed to add dma region\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 		goto unmap_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) unmap_dma:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 		       region->iova, region->length, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unpin_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	afu_dma_unpin_pages(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) free_region:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	kfree(region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)  * afu_dma_unmap_region - unmap dma memory region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)  * @pdata: feature device platform data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)  * @iova: dma address of the region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)  * Unmap dma memory region based on @iova.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)  * Return 0 for success, otherwise error code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) int afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	struct dfl_afu_dma_region *region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	mutex_lock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	region = afu_dma_region_find_iova(pdata, iova);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	if (!region) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 		mutex_unlock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	if (region->in_use) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 		mutex_unlock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 		return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	afu_dma_region_remove(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	mutex_unlock(&pdata->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 		       region->iova, region->length, DMA_BIDIRECTIONAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	afu_dma_unpin_pages(pdata, region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 	kfree(region);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }