^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright 2019 Google LLC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Refer to Documentation/block/inline-encryption.rst for detailed explanation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define pr_fmt(fmt) "blk-crypto-fallback: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <crypto/skcipher.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/blk-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/blk-crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/keyslot-manager.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/mempool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "blk-crypto-internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static unsigned int num_prealloc_bounce_pg = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) module_param(num_prealloc_bounce_pg, uint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) MODULE_PARM_DESC(num_prealloc_bounce_pg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) "Number of preallocated bounce pages for the blk-crypto crypto API fallback");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static unsigned int blk_crypto_num_keyslots = 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) MODULE_PARM_DESC(num_keyslots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) "Number of keyslots for the blk-crypto crypto API fallback");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static unsigned int num_prealloc_fallback_crypt_ctxs = 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) module_param(num_prealloc_fallback_crypt_ctxs, uint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct bio_fallback_crypt_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct bio_crypt_ctx crypt_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * Copy of the bvec_iter when this bio was submitted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * We only want to en/decrypt the part of the bio as described by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * bvec_iter upon submission because bio might be split before being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * resubmitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct bvec_iter crypt_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) void *bi_private_orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) bio_end_io_t *bi_end_io_orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static struct kmem_cache *bio_fallback_crypt_ctx_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static mempool_t *bio_fallback_crypt_ctx_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * all of a mode's tfms when that mode starts being used. Since each mode may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * need all the keyslots at some point, each mode needs its own tfm for each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * match the behavior of real inline encryption hardware (which only supports a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * single encryption context per keyslot), we only allow one tfm per keyslot to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * be used at a time - the rest of the unused tfms have their keys cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static DEFINE_MUTEX(tfms_init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static struct blk_crypto_keyslot {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) enum blk_crypto_mode_num crypto_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) } *blk_crypto_keyslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static struct blk_keyslot_manager blk_crypto_ksm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static struct workqueue_struct *blk_crypto_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static mempool_t *blk_crypto_bounce_page_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * This is the key we set when evicting a keyslot. This *should* be the all 0's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * key, but AES-XTS rejects that key, so we use some random bytes instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static void blk_crypto_evict_keyslot(unsigned int slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /* Clear the key in the skcipher */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) blk_crypto_modes[crypto_mode].keysize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) WARN_ON(err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static int blk_crypto_keyslot_program(struct blk_keyslot_manager *ksm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) const struct blk_crypto_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned int slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) const enum blk_crypto_mode_num crypto_mode =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) key->crypto_cfg.crypto_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (crypto_mode != slotp->crypto_mode &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) blk_crypto_evict_keyslot(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) slotp->crypto_mode = crypto_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) key->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) blk_crypto_evict_keyslot(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static int blk_crypto_keyslot_evict(struct blk_keyslot_manager *ksm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) const struct blk_crypto_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) unsigned int slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) blk_crypto_evict_keyslot(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * The crypto API fallback KSM ops - only used for a bio when it specifies a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * blk_crypto_key that was not supported by the device's inline encryption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static const struct blk_ksm_ll_ops blk_crypto_ksm_ll_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .keyslot_program = blk_crypto_keyslot_program,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .keyslot_evict = blk_crypto_keyslot_evict,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct bio *src_bio = enc_bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) for (i = 0; i < enc_bio->bi_vcnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) mempool_free(enc_bio->bi_io_vec[i].bv_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) blk_crypto_bounce_page_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) src_bio->bi_status = enc_bio->bi_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) bio_put(enc_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) bio_endio(src_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static struct bio *blk_crypto_clone_bio(struct bio *bio_src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (!bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) bio->bi_disk = bio_src->bi_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) bio->bi_opf = bio_src->bi_opf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) bio->bi_ioprio = bio_src->bi_ioprio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) bio->bi_write_hint = bio_src->bi_write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) bio_for_each_segment(bv, bio_src, iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) bio->bi_io_vec[bio->bi_vcnt++] = bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) bio_clone_blkg_association(bio, bio_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) blkcg_bio_issue_init(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) bio_clone_skip_dm_default_key(bio, bio_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static bool blk_crypto_alloc_cipher_req(struct blk_ksm_keyslot *slot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct skcipher_request **ciph_req_ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct crypto_wait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) struct skcipher_request *ciph_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) const struct blk_crypto_keyslot *slotp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) int keyslot_idx = blk_ksm_get_slot_idx(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) slotp = &blk_crypto_keyslots[keyslot_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) if (!ciph_req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) skcipher_request_set_callback(ciph_req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) CRYPTO_TFM_REQ_MAY_BACKLOG |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) CRYPTO_TFM_REQ_MAY_SLEEP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) crypto_req_done, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) *ciph_req_ret = ciph_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static bool blk_crypto_split_bio_if_needed(struct bio **bio_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct bio *bio = *bio_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) unsigned int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned int num_sectors = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) bio_for_each_segment(bv, bio, iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) num_sectors += bv.bv_len >> SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (++i == BIO_MAX_PAGES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (num_sectors < bio_sectors(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct bio *split_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (!split_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) bio_chain(split_bio, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) *bio_ptr = split_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) union blk_crypto_iv {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) u8 bytes[BLK_CRYPTO_MAX_IV_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) union blk_crypto_iv *iv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) iv->dun[i] = cpu_to_le64(dun[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * The crypto API fallback's encryption routine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * Allocate a bounce bio for encryption, encrypt the input bio using crypto API,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * and replace *bio_ptr with the bounce bio. May split input bio if it's too
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * large. Returns true on success. Returns false and sets bio->bi_status on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct bio *src_bio, *enc_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct bio_crypt_ctx *bc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct blk_ksm_keyslot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct skcipher_request *ciph_req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) DECLARE_CRYPTO_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct scatterlist src, dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) union blk_crypto_iv iv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) unsigned int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) blk_status_t blk_st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /* Split the bio if it's too big for single page bvec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!blk_crypto_split_bio_if_needed(bio_ptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) src_bio = *bio_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) bc = src_bio->bi_crypt_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* Allocate bounce bio for encryption */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) enc_bio = blk_crypto_clone_bio(src_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (!enc_bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) src_bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * Use the crypto API fallback keyslot manager to get a crypto_skcipher
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * for the algorithm and key specified for this bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) blk_st = blk_ksm_get_slot_for_key(&blk_crypto_ksm, bc->bc_key, &slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (blk_st != BLK_STS_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) src_bio->bi_status = blk_st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) goto out_put_enc_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) /* and then allocate an skcipher_request for it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (!blk_crypto_alloc_cipher_req(slot, &ciph_req, &wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) src_bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) goto out_release_keyslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) sg_init_table(&src, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) sg_init_table(&dst, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) iv.bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* Encrypt each page in the bounce bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) for (i = 0; i < enc_bio->bi_vcnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct page *plaintext_page = enc_bvec->bv_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct page *ciphertext_page =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) enc_bvec->bv_page = ciphertext_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (!ciphertext_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) src_bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) goto out_free_bounce_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) sg_set_page(&src, plaintext_page, data_unit_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) enc_bvec->bv_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) sg_set_page(&dst, ciphertext_page, data_unit_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) enc_bvec->bv_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) /* Encrypt each data unit in this page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) blk_crypto_dun_to_iv(curr_dun, &iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) &wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) src_bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) goto out_free_bounce_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) bio_crypt_dun_increment(curr_dun, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) src.offset += data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) dst.offset += data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) enc_bio->bi_private = src_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) *bio_ptr = enc_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) enc_bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) goto out_free_ciph_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) out_free_bounce_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) while (i > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) mempool_free(enc_bio->bi_io_vec[--i].bv_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) blk_crypto_bounce_page_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) out_free_ciph_req:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) skcipher_request_free(ciph_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) out_release_keyslot:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) blk_ksm_put_slot(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) out_put_enc_bio:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (enc_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) bio_put(enc_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * The crypto API fallback's main decryption routine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * Decrypts input bio in place, and calls bio_endio on the bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static void blk_crypto_fallback_decrypt_bio(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct bio_fallback_crypt_ctx *f_ctx =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) container_of(work, struct bio_fallback_crypt_ctx, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) struct bio *bio = f_ctx->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct blk_ksm_keyslot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct skcipher_request *ciph_req = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) DECLARE_CRYPTO_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) union blk_crypto_iv iv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct scatterlist sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct bio_vec bv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct bvec_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) blk_status_t blk_st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * Use the crypto API fallback keyslot manager to get a crypto_skcipher
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * for the algorithm and key specified for this bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) blk_st = blk_ksm_get_slot_for_key(&blk_crypto_ksm, bc->bc_key, &slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (blk_st != BLK_STS_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) bio->bi_status = blk_st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) goto out_no_keyslot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* and then allocate an skcipher_request for it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (!blk_crypto_alloc_cipher_req(slot, &ciph_req, &wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) bio->bi_status = BLK_STS_RESOURCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) sg_init_table(&sg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) iv.bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /* Decrypt each segment in the bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct page *page = bv.bv_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) /* Decrypt each data unit in the segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) for (i = 0; i < bv.bv_len; i += data_unit_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) blk_crypto_dun_to_iv(curr_dun, &iv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) &wait)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) bio_crypt_dun_increment(curr_dun, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) sg.offset += data_unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) skcipher_request_free(ciph_req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) blk_ksm_put_slot(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) out_no_keyslot:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) mempool_free(f_ctx, bio_fallback_crypt_ctx_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * blk_crypto_fallback_decrypt_endio - queue bio for fallback decryption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * @bio: the bio to queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * Restore bi_private and bi_end_io, and queue the bio for decryption into a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * workqueue, since this function will be called from an atomic context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static void blk_crypto_fallback_decrypt_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) struct bio_fallback_crypt_ctx *f_ctx = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) bio->bi_private = f_ctx->bi_private_orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) bio->bi_end_io = f_ctx->bi_end_io_orig;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* If there was an IO error, don't queue for decrypt. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (bio->bi_status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) mempool_free(f_ctx, bio_fallback_crypt_ctx_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) INIT_WORK(&f_ctx->work, blk_crypto_fallback_decrypt_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) f_ctx->bio = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) queue_work(blk_crypto_wq, &f_ctx->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * @bio_ptr: pointer to the bio to prepare
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * If bio is doing a WRITE operation, this splits the bio into two parts if it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * too big (see blk_crypto_split_bio_if_needed). It then allocates a bounce bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * for the first part, encrypts it, and update bio_ptr to point to the bounce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) * bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * For a READ operation, we mark the bio for decryption by using bi_private and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * bi_end_io.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * In either case, this function will make the bio look like a regular bio (i.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * as if no encryption context was ever specified) for the purposes of the rest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * of the stack except for blk-integrity (blk-integrity and blk-crypto are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * currently supported together).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * Return: true on success. Sets bio->bi_status and returns false on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) struct bio *bio = *bio_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct bio_crypt_ctx *bc = bio->bi_crypt_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct bio_fallback_crypt_ctx *f_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) /* User didn't call blk_crypto_start_using_key() first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (!blk_ksm_crypto_cfg_supported(&blk_crypto_ksm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) &bc->bc_key->crypto_cfg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) bio->bi_status = BLK_STS_NOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (bio_data_dir(bio) == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return blk_crypto_fallback_encrypt_bio(bio_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * bio READ case: Set up a f_ctx in the bio's bi_private and set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * bi_end_io appropriately to trigger decryption when the bio is ended.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) f_ctx->crypt_ctx = *bc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) f_ctx->crypt_iter = bio->bi_iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) f_ctx->bi_private_orig = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) f_ctx->bi_end_io_orig = bio->bi_end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) bio->bi_private = (void *)f_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) bio->bi_end_io = blk_crypto_fallback_decrypt_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) bio_crypt_free_ctx(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return blk_ksm_evict_key(&blk_crypto_ksm, key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) static bool blk_crypto_fallback_inited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) static int blk_crypto_fallback_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (blk_crypto_fallback_inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) blk_crypto_ksm.ksm_ll_ops = blk_crypto_ksm_ll_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) blk_crypto_ksm.max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) blk_crypto_ksm.features = BLK_CRYPTO_FEATURE_STANDARD_KEYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /* All blk-crypto modes have a crypto API fallback. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) blk_crypto_ksm.crypto_modes_supported[i] = 0xFFFFFFFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) blk_crypto_ksm.crypto_modes_supported[BLK_ENCRYPTION_MODE_INVALID] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) blk_crypto_wq = alloc_workqueue("blk_crypto_wq",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) WQ_UNBOUND | WQ_HIGHPRI |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) WQ_MEM_RECLAIM, num_online_cpus());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (!blk_crypto_wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) goto fail_free_ksm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) sizeof(blk_crypto_keyslots[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (!blk_crypto_keyslots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) goto fail_free_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) blk_crypto_bounce_page_pool =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) mempool_create_page_pool(num_prealloc_bounce_pg, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (!blk_crypto_bounce_page_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) goto fail_free_keyslots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (!bio_fallback_crypt_ctx_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) goto fail_free_bounce_page_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) bio_fallback_crypt_ctx_pool =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) bio_fallback_crypt_ctx_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) if (!bio_fallback_crypt_ctx_pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) goto fail_free_crypt_ctx_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) blk_crypto_fallback_inited = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) fail_free_crypt_ctx_cache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) kmem_cache_destroy(bio_fallback_crypt_ctx_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) fail_free_bounce_page_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) mempool_destroy(blk_crypto_bounce_page_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) fail_free_keyslots:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) kfree(blk_crypto_keyslots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) fail_free_wq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) destroy_workqueue(blk_crypto_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) fail_free_ksm:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) blk_ksm_destroy(&blk_crypto_ksm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * Prepare blk-crypto-fallback for the specified crypto mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * Returns -ENOPKG if the needed crypto API support is missing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) const char *cipher_str = blk_crypto_modes[mode_num].cipher_str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct blk_crypto_keyslot *slotp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * Fast path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * for each i are visible before we try to access them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (likely(smp_load_acquire(&tfms_inited[mode_num])))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) mutex_lock(&tfms_init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (tfms_inited[mode_num])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) err = blk_crypto_fallback_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) for (i = 0; i < blk_crypto_num_keyslots; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) slotp = &blk_crypto_keyslots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (IS_ERR(slotp->tfms[mode_num])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) err = PTR_ERR(slotp->tfms[mode_num]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (err == -ENOENT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) pr_warn_once("Missing crypto API support for \"%s\"\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) cipher_str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) err = -ENOPKG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) slotp->tfms[mode_num] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) goto out_free_tfms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) crypto_skcipher_set_flags(slotp->tfms[mode_num],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * for each i are visible before we set tfms_inited[mode_num].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) smp_store_release(&tfms_inited[mode_num], true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) out_free_tfms:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) for (i = 0; i < blk_crypto_num_keyslots; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) slotp = &blk_crypto_keyslots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) crypto_free_skcipher(slotp->tfms[mode_num]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) slotp->tfms[mode_num] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) mutex_unlock(&tfms_init_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags