^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * OMAP Crypto driver common support routines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2017 Texas Instruments Incorporated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Tero Kristo <t-kristo@ti.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/scatterlist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <crypto/scatterwalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "omap-crypto.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) static int omap_crypto_copy_sg_lists(int total, int bs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct scatterlist **sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) struct scatterlist *new_sg, u16 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) int n = sg_nents(*sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct scatterlist *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) if (!new_sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) sg_init_table(new_sg, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) tmp = new_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) while (*sg && total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) int len = (*sg)->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (total < len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) len = total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (len > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) total -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) sg_set_page(tmp, sg_page(*sg), len, (*sg)->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) if (total <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) sg_mark_end(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) tmp = sg_next(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) *sg = sg_next(*sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) *sg = new_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static int omap_crypto_copy_sgs(int total, int bs, struct scatterlist **sg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct scatterlist *new_sg, u16 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) void *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) int pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) int new_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) new_len = ALIGN(total, bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) pages = get_order(new_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (!buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) pr_err("%s: Couldn't allocate pages for unaligned cases.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (flags & OMAP_CRYPTO_COPY_DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) scatterwalk_map_and_copy(buf, *sg, 0, total, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (flags & OMAP_CRYPTO_ZERO_BUF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) memset(buf + total, 0, new_len - total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) sg_init_table(new_sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) sg_set_buf(new_sg, buf, new_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) *sg = new_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int omap_crypto_check_sg(struct scatterlist *sg, int total, int bs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) u16 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) int num_sg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (!IS_ALIGNED(total, bs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) while (sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) num_sg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (!IS_ALIGNED(sg->offset, 4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (!IS_ALIGNED(sg->length, bs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #ifdef CONFIG_ZONE_DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (page_zonenum(sg_page(sg)) != ZONE_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) len += sg->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) sg = sg_next(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (len >= total)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if ((flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) && num_sg > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) return OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (len != total)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return OMAP_CRYPTO_BAD_DATA_LENGTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct scatterlist *new_sg, u16 flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) u8 flags_shift, unsigned long *dd_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) *dd_flags &= ~(OMAP_CRYPTO_COPY_MASK << flags_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) if (flags & OMAP_CRYPTO_FORCE_COPY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) ret = OMAP_CRYPTO_NOT_ALIGNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) ret = omap_crypto_check_sg(*sg, total, bs, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (ret == OMAP_CRYPTO_NOT_ALIGNED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) ret = omap_crypto_copy_sgs(total, bs, sg, new_sg, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) *dd_flags |= OMAP_CRYPTO_DATA_COPIED << flags_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) } else if (ret == OMAP_CRYPTO_BAD_DATA_LENGTH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) ret = omap_crypto_copy_sg_lists(total, bs, sg, new_sg, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) *dd_flags |= OMAP_CRYPTO_SG_COPIED << flags_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) } else if (flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) sg_set_buf(new_sg, sg_virt(*sg), (*sg)->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) EXPORT_SYMBOL_GPL(omap_crypto_align_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static void omap_crypto_copy_data(struct scatterlist *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct scatterlist *dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) int offset, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int amt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) void *srcb, *dstb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) int srco = 0, dsto = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) while (src && dst && len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (srco >= src->length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) srco -= src->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) src = sg_next(src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (dsto >= dst->length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) dsto -= dst->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) dst = sg_next(dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) amt = min(src->length - srco, dst->length - dsto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) amt = min(len, amt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) srcb = kmap_atomic(sg_page(src)) + srco + src->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) dstb = kmap_atomic(sg_page(dst)) + dsto + dst->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) memcpy(dstb, srcb, amt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (!PageSlab(sg_page(dst)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) flush_kernel_dcache_page(sg_page(dst));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) kunmap_atomic(srcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) kunmap_atomic(dstb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) srco += amt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) dsto += amt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) len -= amt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int offset, int len, u8 flags_shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) void *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) flags >>= flags_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) flags &= OMAP_CRYPTO_COPY_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (!flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) buf = sg_virt(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) pages = get_order(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (orig && (flags & OMAP_CRYPTO_COPY_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) omap_crypto_copy_data(sg, orig, offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (flags & OMAP_CRYPTO_DATA_COPIED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) free_pages((unsigned long)buf, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) else if (flags & OMAP_CRYPTO_SG_COPIED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) kfree(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) EXPORT_SYMBOL_GPL(omap_crypto_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) MODULE_DESCRIPTION("OMAP crypto support library.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) MODULE_AUTHOR("Tero Kristo <t-kristo@ti.com>");