^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * iommu_fill_pdir - Insert coalesced scatter/gather chunks into the I/O Pdir.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * @ioc: The I/O Controller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * @startsg: The scatter/gather list of coalesced chunks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * @nents: The number of entries in the scatter/gather list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * @hint: The DMA Hint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * This function inserts the coalesced scatter/gather list chunks into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * I/O Controller's I/O Pdir.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) static inline unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) unsigned long hint,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) void (*iommu_io_pdir_entry)(u64 *, space_t, unsigned long,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) unsigned long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct scatterlist *dma_sg = startsg; /* pointer to current DMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) unsigned int n_mappings = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) unsigned long dma_offset = 0, dma_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) u64 *pdirp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* Horrible hack. For efficiency's sake, dma_sg starts one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * entry below the true start (it is immediately incremented
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * in the loop) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) dma_sg--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) while (nents-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) unsigned long vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) DBG_RUN_SG(" %d : %08lx/%05x %p/%05x\n", nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) (unsigned long)sg_dma_address(startsg), cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) sg_virt(startsg), startsg->length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) ** Look for the start of a new DMA stream
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) if (sg_dma_address(startsg) & PIDE_FLAG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) BUG_ON(pdirp && (dma_len != sg_dma_len(dma_sg)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) dma_sg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) dma_len = sg_dma_len(startsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) sg_dma_len(startsg) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) dma_offset = (unsigned long) pide & ~IOVP_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) n_mappings++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #if defined(ZX1_SUPPORT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* Pluto IOMMU IO Virt Address is not zero based */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) sg_dma_address(dma_sg) = pide | ioc->ibase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /* SBA, ccio, and dino are zero based.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Trying to save a few CPU cycles for most users.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) sg_dma_address(dma_sg) = pide;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) pdirp = &(ioc->pdir_base[pide >> IOVP_SHIFT]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) prefetchw(pdirp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) BUG_ON(pdirp == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) vaddr = (unsigned long)sg_virt(startsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) sg_dma_len(dma_sg) += startsg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) size = startsg->length + dma_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) dma_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #ifdef IOMMU_MAP_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) ioc->msg_pages += startsg->length >> IOVP_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) iommu_io_pdir_entry(pdirp, KERNEL_SPACE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) vaddr, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) vaddr += IOVP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) size -= IOVP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pdirp++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) } while(unlikely(size > 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) startsg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return(n_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ** First pass is to walk the SG list and determine where the breaks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) ** in the DMA stream. Allocates PDIR entries but does not fill them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ** Returns the number of DMA chunks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) **
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ** Doing the fill separate from the coalescing/allocation keeps the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ** code simpler. Future enhancement could make one pass through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) ** the sglist do both.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static inline unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) iommu_coalesce_chunks(struct ioc *ioc, struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct scatterlist *startsg, int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) int (*iommu_alloc_range)(struct ioc *, struct device *, size_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct scatterlist *contig_sg; /* contig chunk head */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned long dma_offset, dma_len; /* start/len of DMA stream */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned int n_mappings = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) unsigned int max_seg_size = min(dma_get_max_seg_size(dev),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) (unsigned)DMA_CHUNK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned int max_seg_boundary = dma_get_seg_boundary(dev) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (max_seg_boundary) /* check if the addition above didn't overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) max_seg_size = min(max_seg_size, max_seg_boundary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) while (nents > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ** Prepare for first/next DMA stream
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) contig_sg = startsg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) dma_len = startsg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) dma_offset = startsg->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* PARANOID: clear entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) sg_dma_address(startsg) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) sg_dma_len(startsg) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) ** This loop terminates one iteration "early" since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) ** it's always looking one "ahead".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) while(--nents > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) unsigned long prev_end, sg_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) prev_end = (unsigned long)sg_virt(startsg) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) startsg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) startsg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) sg_start = (unsigned long)sg_virt(startsg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* PARANOID: clear entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) sg_dma_address(startsg) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) sg_dma_len(startsg) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) ** First make sure current dma stream won't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ** exceed max_seg_size if we coalesce the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) ** next entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (unlikely(ALIGN(dma_len + dma_offset + startsg->length, IOVP_SIZE) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) max_seg_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * Next see if we can append the next chunk (i.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * it must end on one page and begin on another, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * it must start on the same address as the previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * entry ended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (unlikely((prev_end != sg_start) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) ((prev_end | sg_start) & ~PAGE_MASK)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) dma_len += startsg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) ** End of DMA Stream
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) ** Terminate last VCONTIG block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ** Allocate space for DMA stream.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) sg_dma_len(contig_sg) = dma_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) dma_len = ALIGN(dma_len + dma_offset, IOVP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) sg_dma_address(contig_sg) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) PIDE_FLAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) | (iommu_alloc_range(ioc, dev, dma_len) << IOVP_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) | dma_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) n_mappings++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) return n_mappings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags