^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Scatterlist handling helpers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kmemleak.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * sg_next - return the next scatterlist entry in a list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * @sg: The current sg entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Usually the next entry will be @sg@ + 1, but if this sg element is part
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * of a chained scatterlist, it could jump to the start of a new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * scatterlist array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct scatterlist *sg_next(struct scatterlist *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) if (sg_is_last(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) sg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) if (unlikely(sg_is_chain(sg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) sg = sg_chain_ptr(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) EXPORT_SYMBOL(sg_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * sg_nents - return total count of entries in scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * @sg: The scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * Allows to know how many entries are in sg, taking into acount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * chaining as well
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) int sg_nents(struct scatterlist *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) int nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) for (nents = 0; sg; sg = sg_next(sg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) nents++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) EXPORT_SYMBOL(sg_nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * sg_nents_for_len - return total count of entries in scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * needed to satisfy the supplied length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * @sg: The scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * @len: The total required length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Determines the number of entries in sg that are required to meet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * the supplied length, taking into acount chaining as well
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * the number of sg entries needed, negative error on failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) int sg_nents_for_len(struct scatterlist *sg, u64 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) int nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) u64 total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) for (nents = 0, total = 0; sg; sg = sg_next(sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) nents++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) total += sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (total >= len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) return nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) EXPORT_SYMBOL(sg_nents_for_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * sg_last - return the last scatterlist entry in a list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * @sgl: First entry in the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * @nents: Number of entries in the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * Should only be used casually, it (currently) scans the entire list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * to get the last entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * Note that the @sgl@ pointer passed in need not be the first one,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * the important bit is that @nents@ denotes the number of entries that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * exist from @sgl@.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct scatterlist *sg, *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) for_each_sg(sgl, sg, nents, i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ret = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) BUG_ON(!sg_is_last(ret));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) EXPORT_SYMBOL(sg_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * sg_init_table - Initialize SG table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * @sgl: The SG table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * @nents: Number of entries in table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * Notes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * If this is part of a chained sg table, sg_mark_end() should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * used only on the last table part.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) void sg_init_table(struct scatterlist *sgl, unsigned int nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) memset(sgl, 0, sizeof(*sgl) * nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) sg_init_marker(sgl, nents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) EXPORT_SYMBOL(sg_init_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * sg_init_one - Initialize a single entry sg list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * @sg: SG entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * @buf: Virtual address for IO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * @buflen: IO length
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) sg_init_table(sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) sg_set_buf(sg, buf, buflen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) EXPORT_SYMBOL(sg_init_one);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * The default behaviour of sg_alloc_table() is to use these kmalloc/kfree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * helpers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (nents == SG_MAX_SINGLE_ALLOC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * Kmemleak doesn't track page allocations as they are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * commonly used (in a raw form) for kernel data structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * As we chain together a list of pages and then a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * kmalloc (tracked by kmemleak), in order to for that last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * allocation not to become decoupled (and thus a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * false-positive) we need to inform kmemleak of all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * intermediate allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) void *ptr = (void *) __get_free_page(gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return kmalloc_array(nents, sizeof(struct scatterlist),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static void sg_kfree(struct scatterlist *sg, unsigned int nents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (nents == SG_MAX_SINGLE_ALLOC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) kmemleak_free(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) free_page((unsigned long) sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) kfree(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * __sg_free_table - Free a previously mapped sg table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * @table: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * @max_ents: The maximum number of entries per single scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * @nents_first_chunk: Number of entries int the (preallocated) first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * scatterlist chunk, 0 means no such preallocated first chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * @free_fn: Free function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * Free an sg table previously allocated and setup with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * __sg_alloc_table(). The @max_ents value must be identical to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * that previously used with __sg_alloc_table().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) void __sg_free_table(struct sg_table *table, unsigned int max_ents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned int nents_first_chunk, sg_free_fn *free_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct scatterlist *sgl, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) unsigned curr_max_ents = nents_first_chunk ?: max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (unlikely(!table->sgl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) sgl = table->sgl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) while (table->orig_nents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) unsigned int alloc_size = table->orig_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) unsigned int sg_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * If we have more than max_ents segments left,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * then assign 'next' to the sg table after the current one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * sg_size is then one less than alloc size, since the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * element is the chain pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (alloc_size > curr_max_ents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) next = sg_chain_ptr(&sgl[curr_max_ents - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) alloc_size = curr_max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) sg_size = alloc_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) sg_size = alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) table->orig_nents -= sg_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (nents_first_chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) nents_first_chunk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) free_fn(sgl, alloc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) sgl = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) curr_max_ents = max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) table->sgl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) EXPORT_SYMBOL(__sg_free_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * sg_free_table - Free a previously allocated sg table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * @table: The mapped sg table header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) void sg_free_table(struct sg_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) EXPORT_SYMBOL(sg_free_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * __sg_alloc_table - Allocate and initialize an sg table with given allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * @table: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * @nents: Number of entries in sg list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * @max_ents: The maximum number of entries the allocator returns per call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * @nents_first_chunk: Number of entries int the (preallocated) first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * scatterlist chunk, 0 means no such preallocated chunk provided by user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * @gfp_mask: GFP allocation mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * @alloc_fn: Allocator to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * This function returns a @table @nents long. The allocator is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * defined to return scatterlist chunks of maximum size @max_ents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * Thus if @nents is bigger than @max_ents, the scatterlists will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * chained in units of @max_ents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * Notes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * If this function returns non-0 (eg failure), the caller must call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * __sg_free_table() to cleanup any leftover allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int __sg_alloc_table(struct sg_table *table, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) unsigned int max_ents, struct scatterlist *first_chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) unsigned int nents_first_chunk, gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) sg_alloc_fn *alloc_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) struct scatterlist *sg, *prv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) unsigned int left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) unsigned curr_max_ents = nents_first_chunk ?: max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) unsigned prv_max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) memset(table, 0, sizeof(*table));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (nents == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) #ifdef CONFIG_ARCH_NO_SG_CHAIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (WARN_ON_ONCE(nents > max_ents))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) left = nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) prv = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) unsigned int sg_size, alloc_size = left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (alloc_size > curr_max_ents) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) alloc_size = curr_max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) sg_size = alloc_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) sg_size = alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) left -= sg_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (first_chunk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) sg = first_chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) first_chunk = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) sg = alloc_fn(alloc_size, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (unlikely(!sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * Adjust entry count to reflect that the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * entry of the previous table won't be used for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * linkage. Without this, sg_kfree() may get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * confused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (prv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) table->nents = ++table->orig_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) sg_init_table(sg, alloc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) table->nents = table->orig_nents += sg_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * If this is the first mapping, assign the sg table header.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * If this is not the first mapping, chain previous part.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (prv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) sg_chain(prv, prv_max_ents, sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) table->sgl = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * If no more entries after this one, mark the end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) sg_mark_end(&sg[sg_size - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) prv = sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) prv_max_ents = curr_max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) curr_max_ents = max_ents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) } while (left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) EXPORT_SYMBOL(__sg_alloc_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * sg_alloc_table - Allocate and initialize an sg table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * @table: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * @nents: Number of entries in sg list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * @gfp_mask: GFP allocation mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * Allocate and initialize an sg table. If @nents@ is larger than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * SG_MAX_SINGLE_ALLOC a chained sg table will be setup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) NULL, 0, gfp_mask, sg_kmalloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (unlikely(ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) EXPORT_SYMBOL(sg_alloc_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static struct scatterlist *get_next_sg(struct sg_table *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct scatterlist *cur,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) unsigned long needed_sges,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct scatterlist *new_sg, *next_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) unsigned int alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (cur) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) next_sg = sg_next(cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* Check if last entry should be keeped for chainning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (!sg_is_last(next_sg) || needed_sges == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return next_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) alloc_size = min_t(unsigned long, needed_sges, SG_MAX_SINGLE_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) new_sg = sg_kmalloc(alloc_size, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!new_sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) sg_init_table(new_sg, alloc_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (cur) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) __sg_chain(next_sg, new_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) table->orig_nents += alloc_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) table->sgl = new_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) table->orig_nents = alloc_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) table->nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return new_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * __sg_alloc_table_from_pages - Allocate and initialize an sg table from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * an array of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * @sgt: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * @pages: Pointer to an array of page pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * @n_pages: Number of pages in the pages array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * @offset: Offset from start of the first page to the start of a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * @size: Number of valid bytes in the buffer (after offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * @max_segment: Maximum size of a scatterlist element in bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * @prv: Last populated sge in sgt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * @left_pages: Left pages caller have to set after this call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * @gfp_mask: GFP allocation mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * If @prv is NULL, allocate and initialize an sg table from a list of pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * else reuse the scatterlist passed in at @prv.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * Contiguous ranges of the pages are squashed into a single scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * entry up to the maximum size specified in @max_segment. A user may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * provide an offset at a start and a size of valid data in a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * specified by the page array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * Last SGE in sgt on success, PTR_ERR on otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * The allocation in @sgt must be released by sg_free_table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * Notes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * If this function returns non-0 (eg failure), the caller must call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * sg_free_table() to cleanup any leftover allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct page **pages, unsigned int n_pages, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) unsigned long size, unsigned int max_segment,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) struct scatterlist *prv, unsigned int left_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) unsigned int chunks, cur_page, seg_len, i, prv_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) unsigned int added_nents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) struct scatterlist *s = prv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * The algorithm below requires max_segment to be aligned to PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * otherwise it can overshoot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) max_segment = ALIGN_DOWN(max_segment, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (WARN_ON(max_segment < PAGE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && prv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) return ERR_PTR(-EOPNOTSUPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (prv) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) unsigned long paddr = (page_to_pfn(sg_page(prv)) * PAGE_SIZE +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) prv->offset + prv->length) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (WARN_ON(offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /* Merge contiguous pages into the last SG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) prv_len = prv->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) while (n_pages && page_to_pfn(pages[0]) == paddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (prv->length + PAGE_SIZE > max_segment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) prv->length += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) paddr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) n_pages--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) if (!n_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) /* compute number of contiguous chunks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) chunks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) seg_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) for (i = 1; i < n_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) seg_len += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (seg_len >= max_segment ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) chunks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) seg_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /* merging chunks and putting them into the scatterlist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) cur_page = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) for (i = 0; i < chunks; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) unsigned int j, chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /* look for the end of the current chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) seg_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) for (j = cur_page + 1; j < n_pages; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) seg_len += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (seg_len >= max_segment ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) page_to_pfn(pages[j]) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) page_to_pfn(pages[j - 1]) + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /* Pass how many chunks might be left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) s = get_next_sg(sgt, s, chunks - i + left_pages, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (IS_ERR(s)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * Adjust entry length to be as before function was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (prv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) prv->length = prv_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) sg_set_page(s, pages[cur_page],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) min_t(unsigned long, size, chunk_size), offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) added_nents++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) size -= chunk_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) cur_page = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) sgt->nents += added_nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (!left_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) sg_mark_end(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) EXPORT_SYMBOL(__sg_alloc_table_from_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * sg_alloc_table_from_pages - Allocate and initialize an sg table from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * an array of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * @sgt: The sg table header to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * @pages: Pointer to an array of page pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * @n_pages: Number of pages in the pages array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * @offset: Offset from start of the first page to the start of a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * @size: Number of valid bytes in the buffer (after offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * @gfp_mask: GFP allocation mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * Allocate and initialize an sg table from a list of pages. Contiguous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * ranges of the pages are squashed into a single scatterlist node. A user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * may provide an offset at a start and a size of valid data in a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * specified by the page array. The returned sg table is released by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * sg_free_table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * 0 on success, negative error on failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) unsigned int n_pages, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) unsigned long size, gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, n_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) offset, size, UINT_MAX, NULL, 0, gfp_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) EXPORT_SYMBOL(sg_alloc_table_from_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) #ifdef CONFIG_SGL_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * sgl_alloc_order - allocate a scatterlist and its pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * @length: Length in bytes of the scatterlist. Must be at least one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * @order: Second argument for alloc_pages()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * @chainable: Whether or not to allocate an extra element in the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * for scatterlist chaining purposes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * @gfp: Memory allocation flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * @nent_p: [out] Number of entries in the scatterlist that have pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * Returns: A pointer to an initialized scatterlist or %NULL upon failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct scatterlist *sgl_alloc_order(unsigned long long length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) unsigned int order, bool chainable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) gfp_t gfp, unsigned int *nent_p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) struct scatterlist *sgl, *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) unsigned int nent, nalloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) u32 elem_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) nent = round_up(length, PAGE_SIZE << order) >> (PAGE_SHIFT + order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) /* Check for integer overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) if (length > (nent << (PAGE_SHIFT + order)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) nalloc = nent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (chainable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* Check for integer overflow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (nalloc + 1 < nalloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) nalloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) sgl = kmalloc_array(nalloc, sizeof(struct scatterlist),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) gfp & ~GFP_DMA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (!sgl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) sg_init_table(sgl, nalloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) sg = sgl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) while (length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) elem_len = min_t(u64, length, PAGE_SIZE << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) page = alloc_pages(gfp, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) sgl_free_order(sgl, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) sg_set_page(sg, page, elem_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) length -= elem_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) sg = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) WARN_ONCE(length, "length = %lld\n", length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (nent_p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) *nent_p = nent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return sgl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) EXPORT_SYMBOL(sgl_alloc_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * sgl_alloc - allocate a scatterlist and its pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * @length: Length in bytes of the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * @gfp: Memory allocation flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * @nent_p: [out] Number of entries in the scatterlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * Returns: A pointer to an initialized scatterlist or %NULL upon failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct scatterlist *sgl_alloc(unsigned long long length, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) unsigned int *nent_p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return sgl_alloc_order(length, 0, false, gfp, nent_p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) EXPORT_SYMBOL(sgl_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * sgl_free_n_order - free a scatterlist and its pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * @sgl: Scatterlist with one or more elements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * @nents: Maximum number of elements to free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * @order: Second argument for __free_pages()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * Notes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * - If several scatterlists have been chained and each chain element is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * freed separately then it's essential to set nents correctly to avoid that a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * page would get freed twice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * - All pages in a chained scatterlist can be freed at once by setting @nents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * to a high number.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) void sgl_free_n_order(struct scatterlist *sgl, int nents, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) for_each_sg(sgl, sg, nents, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (!sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) page = sg_page(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) __free_pages(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) kfree(sgl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) EXPORT_SYMBOL(sgl_free_n_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * sgl_free_order - free a scatterlist and its pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * @sgl: Scatterlist with one or more elements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * @order: Second argument for __free_pages()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) void sgl_free_order(struct scatterlist *sgl, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) sgl_free_n_order(sgl, INT_MAX, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) EXPORT_SYMBOL(sgl_free_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * sgl_free - free a scatterlist and its pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * @sgl: Scatterlist with one or more elements
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) void sgl_free(struct scatterlist *sgl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) sgl_free_order(sgl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) EXPORT_SYMBOL(sgl_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) #endif /* CONFIG_SGL_ALLOC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) void __sg_page_iter_start(struct sg_page_iter *piter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) struct scatterlist *sglist, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) unsigned long pgoffset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) piter->__pg_advance = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) piter->__nents = nents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) piter->sg = sglist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) piter->sg_pgoffset = pgoffset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) EXPORT_SYMBOL(__sg_page_iter_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) static int sg_page_count(struct scatterlist *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) bool __sg_page_iter_next(struct sg_page_iter *piter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (!piter->__nents || !piter->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) piter->sg_pgoffset += piter->__pg_advance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) piter->__pg_advance = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) piter->sg_pgoffset -= sg_page_count(piter->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) piter->sg = sg_next(piter->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (!--piter->__nents || !piter->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) EXPORT_SYMBOL(__sg_page_iter_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) static int sg_dma_page_count(struct scatterlist *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return PAGE_ALIGN(sg->offset + sg_dma_len(sg)) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) bool __sg_page_iter_dma_next(struct sg_dma_page_iter *dma_iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) struct sg_page_iter *piter = &dma_iter->base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (!piter->__nents || !piter->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) piter->sg_pgoffset += piter->__pg_advance;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) piter->__pg_advance = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) while (piter->sg_pgoffset >= sg_dma_page_count(piter->sg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) piter->sg_pgoffset -= sg_dma_page_count(piter->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) piter->sg = sg_next(piter->sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (!--piter->__nents || !piter->sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) EXPORT_SYMBOL(__sg_page_iter_dma_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * sg_miter_start - start mapping iteration over a sg list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * @miter: sg mapping iter to be started
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * @sgl: sg list to iterate over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * @nents: number of sg entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * Starts mapping iterator @miter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * Don't care.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) unsigned int nents, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) memset(miter, 0, sizeof(struct sg_mapping_iter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) __sg_page_iter_start(&miter->piter, sgl, nents, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) WARN_ON(!(flags & (SG_MITER_TO_SG | SG_MITER_FROM_SG)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) miter->__flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) EXPORT_SYMBOL(sg_miter_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) static bool sg_miter_get_next_page(struct sg_mapping_iter *miter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (!miter->__remaining) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) struct scatterlist *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (!__sg_page_iter_next(&miter->piter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) sg = miter->piter.sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) miter->__offset &= PAGE_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) miter->__remaining = sg->offset + sg->length -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) (miter->piter.sg_pgoffset << PAGE_SHIFT) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) miter->__offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) miter->__remaining = min_t(unsigned long, miter->__remaining,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) PAGE_SIZE - miter->__offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * sg_miter_skip - reposition mapping iterator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * @miter: sg mapping iter to be skipped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * @offset: number of bytes to plus the current location
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * Sets the offset of @miter to its current location plus @offset bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * If mapping iterator @miter has been proceeded by sg_miter_next(), this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * stops @miter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * Don't care if @miter is stopped, or not proceeded yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * Otherwise, preemption disabled if the SG_MITER_ATOMIC is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * true if @miter contains the valid mapping. false if end of sg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * list is reached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) sg_miter_stop(miter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) while (offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) off_t consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) if (!sg_miter_get_next_page(miter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) consumed = min_t(off_t, offset, miter->__remaining);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) miter->__offset += consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) miter->__remaining -= consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) offset -= consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) EXPORT_SYMBOL(sg_miter_skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * sg_miter_next - proceed mapping iterator to the next mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * @miter: sg mapping iter to proceed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * Proceeds @miter to the next mapping. @miter should have been started
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * using sg_miter_start(). On successful return, @miter->page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * @miter->addr and @miter->length point to the current mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * Preemption disabled if SG_MITER_ATOMIC. Preemption must stay disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * till @miter is stopped. May sleep if !SG_MITER_ATOMIC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * true if @miter contains the next mapping. false if end of sg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * list is reached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) bool sg_miter_next(struct sg_mapping_iter *miter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) sg_miter_stop(miter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * Get to the next page if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * __remaining, __offset is adjusted by sg_miter_stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) if (!sg_miter_get_next_page(miter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) miter->page = sg_page_iter_page(&miter->piter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) miter->consumed = miter->length = miter->__remaining;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (miter->__flags & SG_MITER_ATOMIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) miter->addr = kmap_atomic(miter->page) + miter->__offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) miter->addr = kmap(miter->page) + miter->__offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) EXPORT_SYMBOL(sg_miter_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * sg_miter_stop - stop mapping iteration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * @miter: sg mapping iter to be stopped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * Stops mapping iterator @miter. @miter should have been started
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * using sg_miter_start(). A stopped iteration can be resumed by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * calling sg_miter_next() on it. This is useful when resources (kmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * need to be released during iteration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * Context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * Preemption disabled if the SG_MITER_ATOMIC is set. Don't care
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) void sg_miter_stop(struct sg_mapping_iter *miter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) WARN_ON(miter->consumed > miter->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /* drop resources from the last iteration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (miter->addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) miter->__offset += miter->consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) miter->__remaining -= miter->consumed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if ((miter->__flags & SG_MITER_TO_SG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) !PageSlab(miter->page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) flush_kernel_dcache_page(miter->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if (miter->__flags & SG_MITER_ATOMIC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) WARN_ON_ONCE(preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) kunmap_atomic(miter->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) kunmap(miter->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) miter->page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) miter->addr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) miter->length = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) miter->consumed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) EXPORT_SYMBOL(sg_miter_stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) * sg_copy_buffer - Copy data between a linear buffer and an SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * @buf: Where to copy from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * @buflen: The number of bytes to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * @skip: Number of bytes to skip before copying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * @to_buffer: transfer direction (true == from an sg list to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * buffer, false == from a buffer to an sg list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * Returns the number of copied bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) size_t buflen, off_t skip, bool to_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) unsigned int offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) struct sg_mapping_iter miter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) unsigned int sg_flags = SG_MITER_ATOMIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (to_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) sg_flags |= SG_MITER_FROM_SG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) sg_flags |= SG_MITER_TO_SG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) sg_miter_start(&miter, sgl, nents, sg_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (!sg_miter_skip(&miter, skip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) while ((offset < buflen) && sg_miter_next(&miter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) len = min(miter.length, buflen - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (to_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) memcpy(buf + offset, miter.addr, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) memcpy(miter.addr, buf + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) sg_miter_stop(&miter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) return offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) EXPORT_SYMBOL(sg_copy_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * sg_copy_from_buffer - Copy from a linear buffer to an SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * @buf: Where to copy from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * @buflen: The number of bytes to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * Returns the number of copied bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) const void *buf, size_t buflen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return sg_copy_buffer(sgl, nents, (void *)buf, buflen, 0, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) EXPORT_SYMBOL(sg_copy_from_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * sg_copy_to_buffer - Copy from an SG list to a linear buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * @buf: Where to copy to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * @buflen: The number of bytes to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * Returns the number of copied bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) void *buf, size_t buflen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return sg_copy_buffer(sgl, nents, buf, buflen, 0, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) EXPORT_SYMBOL(sg_copy_to_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * @buf: Where to copy from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) * @buflen: The number of bytes to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * @skip: Number of bytes to skip before copying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * Returns the number of copied bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) const void *buf, size_t buflen, off_t skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) return sg_copy_buffer(sgl, nents, (void *)buf, buflen, skip, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) EXPORT_SYMBOL(sg_pcopy_from_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * sg_pcopy_to_buffer - Copy from an SG list to a linear buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * @buf: Where to copy to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * @buflen: The number of bytes to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * @skip: Number of bytes to skip before copying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * Returns the number of copied bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) void *buf, size_t buflen, off_t skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return sg_copy_buffer(sgl, nents, buf, buflen, skip, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) EXPORT_SYMBOL(sg_pcopy_to_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) * sg_zero_buffer - Zero-out a part of a SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) * @sgl: The SG list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * @nents: Number of SG entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * @buflen: The number of bytes to zero out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * @skip: Number of bytes to skip before zeroing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * Returns the number of bytes zeroed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) size_t buflen, off_t skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) unsigned int offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct sg_mapping_iter miter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) sg_miter_start(&miter, sgl, nents, sg_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (!sg_miter_skip(&miter, skip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) while (offset < buflen && sg_miter_next(&miter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) len = min(miter.length, buflen - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) memset(miter.addr, 0, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) sg_miter_stop(&miter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) EXPORT_SYMBOL(sg_zero_buffer);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate