^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * AES XTS routines supporting VMX In-core instructions on Power 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2015 International Business Machines Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/switch_to.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <crypto/aes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <crypto/internal/skcipher.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <crypto/xts.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "aesp8-ppc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) struct p8_aes_xts_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) struct crypto_skcipher *fallback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct aes_key enc_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct aes_key dec_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct aes_key tweak_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static int p8_aes_xts_init(struct crypto_skcipher *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) struct crypto_skcipher *fallback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) fallback = crypto_alloc_skcipher("xts(aes)", 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) CRYPTO_ALG_NEED_FALLBACK |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) CRYPTO_ALG_ASYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) if (IS_ERR(fallback)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) pr_err("Failed to allocate xts(aes) fallback: %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) PTR_ERR(fallback));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return PTR_ERR(fallback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) crypto_skcipher_reqsize(fallback));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) ctx->fallback = fallback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void p8_aes_xts_exit(struct crypto_skcipher *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) crypto_free_skcipher(ctx->fallback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) ret = xts_verify_key(tfm, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) enable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) disable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return ret ? -EINVAL : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static int p8_aes_xts_crypt(struct skcipher_request *req, int enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) u8 tweak[AES_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (req->cryptlen < AES_BLOCK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!crypto_simd_usable() || (req->cryptlen % XTS_BLOCK_SIZE) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) struct skcipher_request *subreq = skcipher_request_ctx(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *subreq = *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) skcipher_request_set_tfm(subreq, ctx->fallback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return enc ? crypto_skcipher_encrypt(subreq) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) crypto_skcipher_decrypt(subreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ret = skcipher_walk_virt(&walk, req, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) enable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) disable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) while ((nbytes = walk.nbytes) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) enable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) aes_p8_xts_encrypt(walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) walk.dst.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) round_down(nbytes, AES_BLOCK_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) &ctx->enc_key, NULL, tweak);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) aes_p8_xts_decrypt(walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) walk.dst.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) round_down(nbytes, AES_BLOCK_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) &ctx->dec_key, NULL, tweak);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) disable_kernel_vsx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static int p8_aes_xts_encrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return p8_aes_xts_crypt(req, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static int p8_aes_xts_decrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return p8_aes_xts_crypt(req, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct skcipher_alg p8_aes_xts_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) .base.cra_name = "xts(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) .base.cra_driver_name = "p8_aes_xts",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) .base.cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) .base.cra_priority = 2000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .base.cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .setkey = p8_aes_xts_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .encrypt = p8_aes_xts_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .decrypt = p8_aes_xts_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .init = p8_aes_xts_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .exit = p8_aes_xts_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .min_keysize = 2 * AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .max_keysize = 2 * AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) .ivsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags