^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/neon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <crypto/aes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "aes-ce-setkey.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) MODULE_LICENSE("GPL v2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) struct aes_block {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) u8 b[AES_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) asmlinkage u32 __aes_ce_sub(u32 l);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) asmlinkage void __aes_ce_invert(struct aes_block *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) const struct aes_block *in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static int num_rounds(struct crypto_aes_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * # of rounds specified by AES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * 128 bit key 10 rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * 192 bit key 12 rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * 256 bit key 14 rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * => n byte key => 6 + (n/4) rounds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return 6 + ctx->key_length / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) aes_encrypt(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) __aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) aes_decrypt(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) __aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * The AES key schedule round constants
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static u8 const rcon[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) u32 kwords = key_len / sizeof(u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct aes_block *key_enc, *key_dec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (key_len != AES_KEYSIZE_128 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) key_len != AES_KEYSIZE_192 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) key_len != AES_KEYSIZE_256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ctx->key_length = key_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) for (i = 0; i < kwords; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) for (i = 0; i < sizeof(rcon); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) u32 *rki = ctx->key_enc + (i * kwords);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) u32 *rko = rki + kwords;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) rko[1] = rko[0] ^ rki[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) rko[2] = rko[1] ^ rki[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) rko[3] = rko[2] ^ rki[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (key_len == AES_KEYSIZE_192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (i >= 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) rko[4] = rko[3] ^ rki[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) rko[5] = rko[4] ^ rki[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) } else if (key_len == AES_KEYSIZE_256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (i >= 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) rko[5] = rko[4] ^ rki[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) rko[6] = rko[5] ^ rki[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) rko[7] = rko[6] ^ rki[7];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * Generate the decryption keys for the Equivalent Inverse Cipher.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * This involves reversing the order of the round keys, and applying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * the Inverse Mix Columns transformation on all but the first and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * the last one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) key_enc = (struct aes_block *)ctx->key_enc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) key_dec = (struct aes_block *)ctx->key_dec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) j = num_rounds(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) key_dec[0] = key_enc[j];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) for (i = 1, j--; j > 0; i++, j--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) __aes_ce_invert(key_dec + i, key_enc + j);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) key_dec[i] = key_enc[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) EXPORT_SYMBOL(ce_aes_expandkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return ce_aes_expandkey(ctx, in_key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) EXPORT_SYMBOL(ce_aes_setkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static struct crypto_alg aes_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) .cra_name = "aes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) .cra_driver_name = "aes-ce",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .cra_priority = 250,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .cra_ctxsize = sizeof(struct crypto_aes_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .cra_cipher = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .cia_min_keysize = AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) .cia_max_keysize = AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .cia_setkey = ce_aes_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .cia_encrypt = aes_cipher_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) .cia_decrypt = aes_cipher_decrypt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static int __init aes_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return crypto_register_alg(&aes_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static void __exit aes_mod_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) crypto_unregister_alg(&aes_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) module_cpu_feature_match(AES, aes_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) module_exit(aes_mod_exit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags