^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Support for Intel AES-NI instructions. This file contains glue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * code, the real AES implementation is in intel-aes_asm.S.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2008, Intel Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Author: Huang Ying <ying.huang@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * interface for 64-bit kernels.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Authors: Adrian Hoban <adrian.hoban@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Gabriele Paoloni <gabriele.paoloni@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Tadeusz Struk (tadeusz.struk@intel.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Aidan O'Mahony (aidan.o.mahony@intel.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Copyright (c) 2010, Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <crypto/algapi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <crypto/aes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <crypto/ctr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <crypto/b128ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <crypto/gcm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <crypto/xts.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/cpu_device_id.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <crypto/scatterwalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <crypto/internal/aead.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <crypto/internal/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <crypto/internal/skcipher.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/crypto/glue_helper.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define AESNI_ALIGN 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define RFC4106_HASH_SUBKEY_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /* This data is stored at the end of the crypto_tfm struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * It's a type of per "session" data storage location.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * This needs to be 16 byte aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct aesni_rfc4106_gcm_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u8 hash_subkey[16] AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) u8 nonce[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct generic_gcmaes_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) u8 hash_subkey[16] AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct aesni_xts_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define GCM_BLOCK_LEN 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct gcm_context_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* init, update and finalize context data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) u8 aad_hash[GCM_BLOCK_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) u64 aad_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) u64 in_length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) u8 partial_block_enc_key[GCM_BLOCK_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) u8 orig_IV[GCM_BLOCK_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) u8 current_counter[GCM_BLOCK_LEN];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) u64 partial_block_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) u64 unused;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) u8 hash_keys[GCM_BLOCK_LEN * 16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) unsigned int key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) const u8 *in, unsigned int len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) const u8 *in, unsigned int len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define AVX_GEN2_OPTSIZE 640
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define AVX_GEN4_OPTSIZE 4096
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) const u8 *in, unsigned int len, u8 *iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /* asmlinkage void aesni_gcm_enc()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * struct gcm_context_data. May be uninitialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * u8 *out, Ciphertext output. Encrypt in-place is allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * const u8 *in, Plaintext input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * unsigned long plaintext_len, Length of data in bytes for encryption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * 16-byte aligned pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * const u8 *aad, Additional Authentication Data (AAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * unsigned long aad_len, Length of AAD in bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * u8 *auth_tag, Authenticated Tag output.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Valid values are 16 (most likely), 12 or 8.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) asmlinkage void aesni_gcm_enc(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) const u8 *in, unsigned long plaintext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* asmlinkage void aesni_gcm_dec()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * struct gcm_context_data. May be uninitialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * u8 *out, Plaintext output. Decrypt in-place is allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * const u8 *in, Ciphertext input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * unsigned long ciphertext_len, Length of data in bytes for decryption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * 16-byte aligned pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * const u8 *aad, Additional Authentication Data (AAD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * to be 8 or 12 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * u8 *auth_tag, Authenticated Tag output.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * Valid values are 16 (most likely), 12 or 8.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) asmlinkage void aesni_gcm_dec(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) const u8 *in, unsigned long ciphertext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /* Scatter / Gather routines, with args similar to above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) asmlinkage void aesni_gcm_init(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) u8 *hash_subkey, const u8 *aad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) unsigned long aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) asmlinkage void aesni_gcm_enc_update(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) const u8 *in, unsigned long plaintext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) asmlinkage void aesni_gcm_dec_update(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) const u8 *in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned long ciphertext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) asmlinkage void aesni_gcm_finalize(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static const struct aesni_gcm_tfm_s {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) const u8 *in, unsigned long plaintext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) const u8 *in, unsigned long ciphertext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void (*finalize)(void *ctx, struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) } *aesni_gcm_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) .init = &aesni_gcm_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) .enc_update = &aesni_gcm_enc_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) .dec_update = &aesni_gcm_dec_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) .finalize = &aesni_gcm_finalize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) void *keys, u8 *out, unsigned int num_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) void *keys, u8 *out, unsigned int num_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) void *keys, u8 *out, unsigned int num_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * asmlinkage void aesni_gcm_init_avx_gen2()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * gcm_data *my_ctx_data, context data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) u8 *hash_subkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) const u8 *aad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) unsigned long aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) const u8 *in, unsigned long plaintext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) const u8 *in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned long ciphertext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) const u8 *in, unsigned long plaintext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) const u8 *in, unsigned long ciphertext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) .init = &aesni_gcm_init_avx_gen2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) .enc_update = &aesni_gcm_enc_update_avx_gen2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) .dec_update = &aesni_gcm_dec_update_avx_gen2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) .finalize = &aesni_gcm_finalize_avx_gen2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * asmlinkage void aesni_gcm_init_avx_gen4()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * gcm_data *my_ctx_data, context data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) u8 *hash_subkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) const u8 *aad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) unsigned long aad_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) const u8 *in, unsigned long plaintext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) const u8 *in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned long ciphertext_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct gcm_context_data *gdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) const u8 *in, unsigned long plaintext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct gcm_context_data *gdata, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) const u8 *in, unsigned long ciphertext_len, u8 *iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) const u8 *aad, unsigned long aad_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) u8 *auth_tag, unsigned long auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) .init = &aesni_gcm_init_avx_gen4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) .enc_update = &aesni_gcm_enc_update_avx_gen4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) .dec_update = &aesni_gcm_dec_update_avx_gen4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) .finalize = &aesni_gcm_finalize_avx_gen4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static inline struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned long align = AESNI_ALIGN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (align <= crypto_tfm_ctx_alignment())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) align = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return PTR_ALIGN(crypto_aead_ctx(tfm), align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) static inline struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long align = AESNI_ALIGN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (align <= crypto_tfm_ctx_alignment())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) align = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return PTR_ALIGN(crypto_aead_ctx(tfm), align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) unsigned long addr = (unsigned long)raw_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) unsigned long align = AESNI_ALIGN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (align <= crypto_tfm_ctx_alignment())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) align = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return (struct crypto_aes_ctx *)ALIGN(addr, align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) const u8 *in_key, unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) key_len != AES_KEYSIZE_256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (!crypto_simd_usable())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) err = aes_expandkey(ctx, in_key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) err = aesni_set_key(ctx, in_key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) aes_encrypt(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) aesni_enc(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (!crypto_simd_usable()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) aes_decrypt(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) aesni_dec(ctx, dst, src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) unsigned int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return aes_set_key_common(crypto_skcipher_tfm(tfm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) crypto_skcipher_ctx(tfm), key, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static int ecb_encrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) err = skcipher_walk_virt(&walk, req, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) while ((nbytes = walk.nbytes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) nbytes & AES_BLOCK_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) nbytes &= AES_BLOCK_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) err = skcipher_walk_done(&walk, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static int ecb_decrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) err = skcipher_walk_virt(&walk, req, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) while ((nbytes = walk.nbytes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) nbytes & AES_BLOCK_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) nbytes &= AES_BLOCK_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) err = skcipher_walk_done(&walk, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static int cbc_encrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) err = skcipher_walk_virt(&walk, req, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) while ((nbytes = walk.nbytes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) nbytes & AES_BLOCK_MASK, walk.iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) nbytes &= AES_BLOCK_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) err = skcipher_walk_done(&walk, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static int cbc_decrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) err = skcipher_walk_virt(&walk, req, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) while ((nbytes = walk.nbytes)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) nbytes & AES_BLOCK_MASK, walk.iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) nbytes &= AES_BLOCK_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) err = skcipher_walk_done(&walk, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct skcipher_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) u8 *ctrblk = walk->iv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) u8 keystream[AES_BLOCK_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) u8 *src = walk->src.virt.addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) u8 *dst = walk->dst.virt.addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) unsigned int nbytes = walk->nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) aesni_enc(ctx, keystream, ctrblk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) crypto_xor_cpy(dst, keystream, src, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) crypto_inc(ctrblk, AES_BLOCK_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) const u8 *in, unsigned int len, u8 *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * based on key length, override with the by8 version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * of ctr mode encryption/decryption for improved performance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * aes_set_key_common() ensures that key length is one of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * {128,192,256}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (ctx->key_length == AES_KEYSIZE_128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) else if (ctx->key_length == AES_KEYSIZE_192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static int ctr_crypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) struct skcipher_walk walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) unsigned int nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) err = skcipher_walk_virt(&walk, req, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) nbytes & AES_BLOCK_MASK, walk.iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) nbytes &= AES_BLOCK_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) err = skcipher_walk_done(&walk, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (walk.nbytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) ctr_crypt_final(ctx, &walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) err = skcipher_walk_done(&walk, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) unsigned int keylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) err = xts_verify_key(tfm, key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) keylen /= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) /* first half of xts-key is for crypt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) key, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* second half of xts-key is for tweak */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) key + keylen, keylen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static void aesni_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_enc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) static void aesni_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_dec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static void aesni_xts_enc32(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) aesni_xts_encrypt(ctx, dst, src, 32 * AES_BLOCK_SIZE, (u8 *)iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static void aesni_xts_dec32(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) aesni_xts_decrypt(ctx, dst, src, 32 * AES_BLOCK_SIZE, (u8 *)iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static const struct common_glue_ctx aesni_enc_xts = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) .num_funcs = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) .fpu_blocks_limit = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) .funcs = { {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) .num_blocks = 32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) .fn_u = { .xts = aesni_xts_enc32 }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) .num_blocks = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) .fn_u = { .xts = aesni_xts_enc }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) } }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static const struct common_glue_ctx aesni_dec_xts = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) .num_funcs = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) .fpu_blocks_limit = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .funcs = { {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .num_blocks = 32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) .fn_u = { .xts = aesni_xts_dec32 }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) .num_blocks = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) .fn_u = { .xts = aesni_xts_dec }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) } }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) static int xts_encrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return glue_xts_req_128bit(&aesni_enc_xts, req, aesni_enc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) aes_ctx(ctx->raw_tweak_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) aes_ctx(ctx->raw_crypt_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) static int xts_decrypt(struct skcipher_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return glue_xts_req_128bit(&aesni_dec_xts, req, aesni_enc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) aes_ctx(ctx->raw_tweak_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) aes_ctx(ctx->raw_crypt_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) struct crypto_aes_ctx ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ret = aes_expandkey(&ctx, key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) /* Clear the data in the hash sub key container to zero.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) /* We want to cipher all zeros to create the hash sub key. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) aes_encrypt(&ctx, hash_subkey, hash_subkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) memzero_explicit(&ctx, sizeof(ctx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (key_len < 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) /*Account for 4 byte nonce at the end.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) key_len -= 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) return aes_set_key_common(crypto_aead_tfm(aead),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) &ctx->aes_key_expanded, key, key_len) ?:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) /* This is the Integrity Check Value (aka the authentication tag) length and can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * be 8, 12 or 16 bytes long. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static int common_rfc4106_set_authsize(struct crypto_aead *aead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) unsigned int authsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) switch (authsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) case 12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) case 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) unsigned int authsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) switch (authsize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) case 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) case 12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) case 13:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) case 14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) case 15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) case 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) unsigned int assoclen, u8 *hash_subkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) u8 *iv, void *aes_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct crypto_aead *tfm = crypto_aead_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) unsigned long auth_tag_len = crypto_aead_authsize(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) struct scatter_walk dst_sg_walk = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) unsigned long left = req->cryptlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) unsigned long len, srclen, dstlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct scatter_walk assoc_sg_walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) struct scatter_walk src_sg_walk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) struct scatterlist src_start[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) struct scatterlist dst_start[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct scatterlist *src_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct scatterlist *dst_sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) u8 *src, *dst, *assoc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) u8 *assocmem = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) u8 authTag[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (!enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) left -= auth_tag_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) gcm_tfm = &aesni_gcm_tfm_avx_gen2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) gcm_tfm = &aesni_gcm_tfm_sse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* Linearize assoc, if not already linear */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (req->src->length >= assoclen && req->src->length &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) (!PageHighMem(sg_page(req->src)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) req->src->offset + req->src->length <= PAGE_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) scatterwalk_start(&assoc_sg_walk, req->src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) assoc = scatterwalk_map(&assoc_sg_walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) /* assoc can be any length, so must be on heap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) assocmem = kmalloc(assoclen, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (unlikely(!assocmem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) assoc = assocmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) scatterwalk_start(&src_sg_walk, src_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) if (req->src != req->dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) dst_sg = scatterwalk_ffwd(dst_start, req->dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) req->assoclen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) scatterwalk_start(&dst_sg_walk, dst_sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) gcm_tfm->init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (req->src != req->dst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) while (left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) src = scatterwalk_map(&src_sg_walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) dst = scatterwalk_map(&dst_sg_walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) srclen = scatterwalk_clamp(&src_sg_walk, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) dstlen = scatterwalk_clamp(&dst_sg_walk, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) len = min(srclen, dstlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) if (enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) gcm_tfm->enc_update(aes_ctx, data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) dst, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) gcm_tfm->dec_update(aes_ctx, data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) dst, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) left -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) scatterwalk_unmap(src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) scatterwalk_unmap(dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) scatterwalk_advance(&src_sg_walk, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) scatterwalk_advance(&dst_sg_walk, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) scatterwalk_done(&src_sg_walk, 0, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) scatterwalk_done(&dst_sg_walk, 1, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) while (left) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) dst = src = scatterwalk_map(&src_sg_walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) len = scatterwalk_clamp(&src_sg_walk, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (enc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) gcm_tfm->enc_update(aes_ctx, data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) src, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) gcm_tfm->dec_update(aes_ctx, data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) src, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) left -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) scatterwalk_unmap(src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) scatterwalk_advance(&src_sg_walk, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) scatterwalk_done(&src_sg_walk, 1, left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) gcm_tfm->finalize(aes_ctx, data, authTag, auth_tag_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (!assocmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) scatterwalk_unmap(assoc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) kfree(assocmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (!enc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) u8 authTagMsg[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /* Copy out original authTag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) scatterwalk_map_and_copy(authTagMsg, req->src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) req->assoclen + req->cryptlen -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) auth_tag_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) auth_tag_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /* Compare generated tag with passed in tag. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) -EBADMSG : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) /* Copy in the authTag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) scatterwalk_map_and_copy(authTag, req->dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) req->assoclen + req->cryptlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) auth_tag_len, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) u8 *hash_subkey, u8 *iv, void *aes_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) u8 *hash_subkey, u8 *iv, void *aes_ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) static int helper_rfc4106_encrypt(struct aead_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct crypto_aead *tfm = crypto_aead_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) void *aes_ctx = &(ctx->aes_key_expanded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) __be32 counter = cpu_to_be32(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /* Assuming we are supporting rfc4106 64-bit extended */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) /* sequence numbers We need to have the AAD length equal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /* to 16 or 20 bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (unlikely(req->assoclen != 16 && req->assoclen != 20))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) /* IV below built */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) for (i = 0; i < 4; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) *(iv+i) = ctx->nonce[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) for (i = 0; i < 8; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) *(iv+4+i) = req->iv[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) *((__be32 *)(iv+12)) = counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) static int helper_rfc4106_decrypt(struct aead_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) __be32 counter = cpu_to_be32(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) struct crypto_aead *tfm = crypto_aead_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) void *aes_ctx = &(ctx->aes_key_expanded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (unlikely(req->assoclen != 16 && req->assoclen != 20))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) /* Assuming we are supporting rfc4106 64-bit extended */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) /* sequence numbers We need to have the AAD length */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /* equal to 16 or 20 bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /* IV below built */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) for (i = 0; i < 4; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) *(iv+i) = ctx->nonce[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) for (i = 0; i < 8; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) *(iv+4+i) = req->iv[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) *((__be32 *)(iv+12)) = counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) static struct crypto_alg aesni_cipher_alg = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) .cra_name = "aes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) .cra_driver_name = "aes-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) .cra_priority = 300,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) .cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) .cra_u = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) .cipher = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) .cia_min_keysize = AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) .cia_max_keysize = AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) .cia_setkey = aes_set_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) .cia_encrypt = aesni_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) .cia_decrypt = aesni_decrypt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) static struct skcipher_alg aesni_skciphers[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) .cra_name = "__ecb(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) .cra_driver_name = "__ecb-aes-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) .cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) .min_keysize = AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) .max_keysize = AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) .setkey = aesni_skcipher_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) .encrypt = ecb_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) .decrypt = ecb_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) .cra_name = "__cbc(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) .cra_driver_name = "__cbc-aes-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) .cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) .min_keysize = AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) .max_keysize = AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) .ivsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) .setkey = aesni_skcipher_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) .encrypt = cbc_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) .decrypt = cbc_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) .cra_name = "__ctr(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) .cra_driver_name = "__ctr-aes-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) .cra_blocksize = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) .min_keysize = AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) .max_keysize = AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) .ivsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) .chunksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) .setkey = aesni_skcipher_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) .encrypt = ctr_crypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) .decrypt = ctr_crypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) .cra_name = "__xts(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) .cra_driver_name = "__xts-aes-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) .cra_priority = 401,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) .cra_blocksize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) .cra_ctxsize = XTS_AES_CTX_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) .min_keysize = 2 * AES_MIN_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) .max_keysize = 2 * AES_MAX_KEY_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) .ivsize = AES_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) .setkey = xts_aesni_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) .encrypt = xts_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) .decrypt = xts_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) static
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) unsigned int key_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) return aes_set_key_common(crypto_aead_tfm(aead),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) &ctx->aes_key_expanded, key, key_len) ?:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) static int generic_gcmaes_encrypt(struct aead_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) struct crypto_aead *tfm = crypto_aead_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) void *aes_ctx = &(ctx->aes_key_expanded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) __be32 counter = cpu_to_be32(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) memcpy(iv, req->iv, 12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) *((__be32 *)(iv+12)) = counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) static int generic_gcmaes_decrypt(struct aead_request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) __be32 counter = cpu_to_be32(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) struct crypto_aead *tfm = crypto_aead_reqtfm(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) void *aes_ctx = &(ctx->aes_key_expanded);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) memcpy(iv, req->iv, 12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) *((__be32 *)(iv+12)) = counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) aes_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) static struct aead_alg aesni_aeads[] = { {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) .setkey = common_rfc4106_set_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) .setauthsize = common_rfc4106_set_authsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) .encrypt = helper_rfc4106_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) .decrypt = helper_rfc4106_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) .ivsize = GCM_RFC4106_IV_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) .maxauthsize = 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) .cra_name = "__rfc4106(gcm(aes))",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) .cra_driver_name = "__rfc4106-gcm-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) .cra_blocksize = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) .cra_alignmask = AESNI_ALIGN - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) }, {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) .setkey = generic_gcmaes_set_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) .setauthsize = generic_gcmaes_set_authsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) .encrypt = generic_gcmaes_encrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) .decrypt = generic_gcmaes_decrypt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) .ivsize = GCM_AES_IV_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) .maxauthsize = 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) .base = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) .cra_name = "__gcm(aes)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) .cra_driver_name = "__generic-gcm-aesni",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) .cra_priority = 400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) .cra_flags = CRYPTO_ALG_INTERNAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) .cra_blocksize = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) .cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) .cra_alignmask = AESNI_ALIGN - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) .cra_module = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) } };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) static struct aead_alg aesni_aeads[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static const struct x86_cpu_id aesni_cpu_id[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) static int __init aesni_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (!x86_match_cpu(aesni_cpu_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (boot_cpu_has(X86_FEATURE_AVX2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) pr_info("AVX2 version of gcm_enc/dec engaged.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (boot_cpu_has(X86_FEATURE_AVX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) pr_info("AVX version of gcm_enc/dec engaged.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) pr_info("SSE version of gcm_enc/dec engaged.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) aesni_gcm_tfm = &aesni_gcm_tfm_sse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) aesni_ctr_enc_tfm = aesni_ctr_enc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) if (boot_cpu_has(X86_FEATURE_AVX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /* optimize performance of ctr mode encryption transform */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) pr_info("AES CTR mode by8 optimization enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) err = crypto_register_alg(&aesni_cipher_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) err = simd_register_skciphers_compat(aesni_skciphers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) ARRAY_SIZE(aesni_skciphers),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) aesni_simd_skciphers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) goto unregister_cipher;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) aesni_simd_aeads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) goto unregister_skciphers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) unregister_skciphers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) aesni_simd_skciphers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) unregister_cipher:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) crypto_unregister_alg(&aesni_cipher_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) static void __exit aesni_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) aesni_simd_aeads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) aesni_simd_skciphers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) crypto_unregister_alg(&aesni_cipher_alg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) late_initcall(aesni_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) module_exit(aesni_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) MODULE_ALIAS_CRYPTO("aes");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags