Orange Pi5 kernel

/* SPDX-License-Identifier: GPL-2.0 */

/* Copyright (c) 2018 Rockchip Electronics Co. Ltd. */

#ifndef __RK_CRYPTO_CORE_H__
#define __RK_CRYPTO_CORE_H__

#include <crypto/aes.h>
#include <crypto/des.h>
#include <crypto/algapi.h>
#include <crypto/md5.h>
#include <crypto/sha.h>
#include <crypto/sm3.h>
#include <crypto/sm4.h>
#include <crypto/gcm.h>
#include <crypto/skcipher.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/rsa.h>
#include <crypto/internal/des.h>
#include <crypto/internal/skcipher.h>

#include <linux/interrupt.h>
#include <linux/delay.h>
#include <linux/clk.h>
#include <linux/io.h>
#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
#include <linux/timer.h>

#include "rk_crypto_bignum.h"

/*
 * Change to the lowest priority, and hardware encryption is
 * invoked explicitly only at the User layer.
 */
#define RK_CRYPTO_PRIORITY		0

/*  Increase the addr_vir buffer size from 1 to 8 pages */
#define RK_BUFFER_ORDER			3
#define RK_BUFFER_SIZE			(PAGE_SIZE << RK_BUFFER_ORDER)

#define RK_DMA_ALIGNMENT		128
#define sha384_state			sha512_state
#define sha224_state			sha256_state

#define RK_FLAG_FINAL			BIT(0)
#define RK_FLAG_UPDATE			BIT(1)

struct rk_crypto_stat {
	unsigned long long	busy_cnt;
	unsigned long long	equeue_cnt;
	unsigned long long	dequeue_cnt;
	unsigned long long	complete_cnt;
	unsigned long long	done_cnt;
	unsigned long long	fake_cnt;
	unsigned long long	irq_cnt;
	unsigned long long	timeout_cnt;
	unsigned long long	error_cnt;
	unsigned long long	ever_queue_max;
	int			last_error;
};

struct rk_crypto_dev {
	struct device			*dev;
	struct reset_control		*rst;
	void __iomem			*reg;
	void __iomem			*pka_reg;
	int				irq;
	struct crypto_queue		queue;
	struct tasklet_struct		queue_task;
	struct tasklet_struct		done_task;
	int				err;
	void				*hw_info;
	struct rk_crypto_soc_data	*soc_data;
	int clks_num;
	struct clk_bulk_data		*clk_bulks;
	const char			*name;
	struct proc_dir_entry		*procfs;
	struct rk_crypto_stat		stat;

	/* device lock */
	spinlock_t			lock;

	/* the public variable */
	struct crypto_async_request	*async_req;
	void				*addr_vir;
	u32				vir_max;
	void				*addr_aad;
	int				aad_max;
	struct scatterlist		src[2];
	struct scatterlist		dst[2];

	struct timer_list		timer;
	bool				busy;
	void (*request_crypto)(struct rk_crypto_dev *rk_dev, const char *name);
	void (*release_crypto)(struct rk_crypto_dev *rk_dev, const char *name);
	int (*load_data)(struct rk_crypto_dev *rk_dev,
			 struct scatterlist *sg_src,
			 struct scatterlist *sg_dst);
	int (*unload_data)(struct rk_crypto_dev *rk_dev);
	int (*enqueue)(struct rk_crypto_dev *rk_dev,
		       struct crypto_async_request *async_req);
};

struct rk_crypto_soc_data {
	const char			*crypto_ver;
	char				**valid_algs_name;
	int				valid_algs_num;
	unsigned int			hw_info_size;
	bool				use_soft_aes192;
	int				default_pka_offset;
	bool				use_lli_chain;

	int (*hw_init)(struct device *dev, void *hw_info);
	void (*hw_deinit)(struct device *dev, void *hw_info);
	const char * const *(*hw_get_rsts)(uint32_t *num);
	struct rk_crypto_algt **(*hw_get_algts)(uint32_t *num);
	bool (*hw_is_algo_valid)(struct rk_crypto_dev *rk_dev,
				 struct rk_crypto_algt *aglt);
};

struct rk_alg_ops {
	int (*start)(struct rk_crypto_dev *rk_dev);
	int (*update)(struct rk_crypto_dev *rk_dev);
	void (*complete)(struct crypto_async_request *base, int err);
	int (*irq_handle)(int irq, void *dev_id);

	int (*hw_write_key)(struct rk_crypto_dev *rk_dev, const u8 *key, u32 key_len);
	void (*hw_write_iv)(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len);
	int (*hw_init)(struct rk_crypto_dev *rk_dev, u32 algo, u32 type);
	int (*hw_dma_start)(struct rk_crypto_dev *rk_dev, uint32_t flag);
	int (*hw_get_result)(struct rk_crypto_dev *rk_dev, uint8_t *data, uint32_t data_len);
};

struct rk_alg_ctx {
	struct rk_alg_ops		ops;
	struct scatterlist		*sg_src;
	struct scatterlist		*sg_dst;
	struct scatterlist		sg_tmp;
	struct scatterlist		sg_aad;
	struct scatterlist		*req_src;
	struct scatterlist		*req_dst;
	size_t				src_nents;
	size_t				dst_nents;
	size_t				map_nents;

	int				is_aead;
	unsigned int			total;
	unsigned int			assoclen;
	unsigned int			count;
	unsigned int			left_bytes;

	dma_addr_t			addr_in;
	dma_addr_t			addr_out;
	dma_addr_t			addr_aad_in;

	bool				aligned;
	bool				is_dma;
	int				align_size;
	int				chunk_size;
};

/* the private variable of hash */
struct rk_ahash_ctx {
	struct rk_alg_ctx		algs_ctx;
	struct rk_crypto_dev		*rk_dev;
	u8				authkey[SHA512_BLOCK_SIZE];
	u32				authkey_len;
	struct scatterlist		hash_sg[2];
	u8				*hash_tmp;
	u32				hash_tmp_len;
	bool				hash_tmp_mapped;
	u32				calc_cnt;

	u8				lastc[RK_DMA_ALIGNMENT];
	u32				lastc_len;

	void				*priv;

	/* for fallback */
	struct crypto_ahash		*fallback_tfm;
};

/* the privete variable of hash for fallback */
struct rk_ahash_rctx {
	struct ahash_request		fallback_req;
	u32				mode;
	u32				flag;
};

/* the private variable of cipher */
struct rk_cipher_ctx {
	struct rk_alg_ctx		algs_ctx;
	struct rk_crypto_dev		*rk_dev;
	unsigned char			key[AES_MAX_KEY_SIZE * 2];
	unsigned int			keylen;
	u32				mode;
	u8				iv[AES_BLOCK_SIZE];
	u32				iv_len;
	u8				lastc[AES_BLOCK_SIZE];
	bool				is_enc;
	void				*priv;

	/* for fallback */
	bool				fallback_key_inited;
	struct crypto_skcipher		*fallback_tfm;
	struct skcipher_request		fallback_req;	// keep at the end
	struct crypto_aead		*fallback_aead;
};

struct rk_rsa_ctx {
	struct rk_alg_ctx		algs_ctx;
	struct rk_bignum *n;
	struct rk_bignum *e;
	struct rk_bignum *d;

	struct rk_crypto_dev		*rk_dev;
};

enum alg_type {
	ALG_TYPE_HASH,
	ALG_TYPE_HMAC,
	ALG_TYPE_CIPHER,
	ALG_TYPE_ASYM,
	ALG_TYPE_AEAD,
	ALG_TYPE_MAX,
};

struct rk_crypto_algt {
	struct rk_crypto_dev		*rk_dev;
	union {
		struct skcipher_alg	crypto;
		struct ahash_alg	hash;
		struct akcipher_alg	asym;
		struct aead_alg		aead;
	} alg;
	enum alg_type			type;
	u32				algo;
	u32				mode;
	char				*name;
	bool				use_soft_aes192;
	bool				valid_flag;
};

enum rk_hash_algo {
	HASH_ALGO_MD5,
	HASH_ALGO_SHA1,
	HASH_ALGO_SHA224,
	HASH_ALGO_SHA256,
	HASH_ALGO_SHA384,
	HASH_ALGO_SHA512,
	HASH_ALGO_SM3,
	HASH_ALGO_SHA512_224,
	HASH_ALGO_SHA512_256,
};

enum rk_cipher_algo {
	CIPHER_ALGO_DES,
	CIPHER_ALGO_DES3_EDE,
	CIPHER_ALGO_AES,
	CIPHER_ALGO_SM4,
};

enum rk_cipher_mode {
	CIPHER_MODE_ECB,
	CIPHER_MODE_CBC,
	CIPHER_MODE_CFB,
	CIPHER_MODE_OFB,
	CIPHER_MODE_CTR,
	CIPHER_MODE_XTS,
	CIPHER_MODE_CTS,
	CIPHER_MODE_CCM,
	CIPHER_MODE_GCM,
	CIPHER_MODE_CMAC,
	CIPHER_MODE_CBCMAC,
};

#define DES_MIN_KEY_SIZE	DES_KEY_SIZE
#define DES_MAX_KEY_SIZE	DES_KEY_SIZE
#define DES3_EDE_MIN_KEY_SIZE	DES3_EDE_KEY_SIZE
#define DES3_EDE_MAX_KEY_SIZE	DES3_EDE_KEY_SIZE
#define SM4_MIN_KEY_SIZE	SM4_KEY_SIZE
#define SM4_MAX_KEY_SIZE	SM4_KEY_SIZE

#define MD5_BLOCK_SIZE		SHA1_BLOCK_SIZE

#define  RK_AEAD_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\
	.name = #algo_name,\
	.type = ALG_TYPE_AEAD,\
	.algo = CIPHER_ALGO_##cipher_algo,\
	.mode = CIPHER_MODE_##cipher_mode,\
	.alg.aead = {\
		.base.cra_name		= #algo_name,\
		.base.cra_driver_name	= #driver_name,\
		.base.cra_priority	= RK_CRYPTO_PRIORITY,\
		.base.cra_flags		= CRYPTO_ALG_TYPE_AEAD |\
					  CRYPTO_ALG_KERN_DRIVER_ONLY |\
					  CRYPTO_ALG_ASYNC |\
					  CRYPTO_ALG_NEED_FALLBACK,\
		.base.cra_blocksize	= 1,\
		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),\
		.base.cra_alignmask	= 0x07,\
		.base.cra_module	= THIS_MODULE,\
		.init		= rk_aead_init_tfm,\
		.exit		= rk_aead_exit_tfm,\
		.ivsize		= GCM_AES_IV_SIZE,\
		.chunksize      = cipher_algo##_BLOCK_SIZE,\
		.maxauthsize    = AES_BLOCK_SIZE,\
		.setkey		= rk_aead_setkey,\
		.setauthsize	= rk_aead_gcm_setauthsize,\
		.encrypt	= rk_aead_encrypt,\
		.decrypt	= rk_aead_decrypt,\
	} \
}

#define  RK_CIPHER_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\
	.name = #algo_name,\
	.type = ALG_TYPE_CIPHER,\
	.algo = CIPHER_ALGO_##cipher_algo,\
	.mode = CIPHER_MODE_##cipher_mode,\
	.alg.crypto = {\
		.base.cra_name		= #algo_name,\
		.base.cra_driver_name	= #driver_name,\
		.base.cra_priority	= RK_CRYPTO_PRIORITY,\
		.base.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |\
					  CRYPTO_ALG_ASYNC |\
					  CRYPTO_ALG_NEED_FALLBACK,\
		.base.cra_blocksize	= cipher_algo##_BLOCK_SIZE,\
		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),\
		.base.cra_alignmask	= 0x07,\
		.base.cra_module	= THIS_MODULE,\
		.init		= rk_ablk_init_tfm,\
		.exit		= rk_ablk_exit_tfm,\
		.min_keysize	= cipher_algo##_MIN_KEY_SIZE,\
		.max_keysize	= cipher_algo##_MAX_KEY_SIZE,\
		.ivsize		= cipher_algo##_BLOCK_SIZE,\
		.chunksize      = cipher_algo##_BLOCK_SIZE,\
		.setkey		= rk_cipher_setkey,\
		.encrypt	= rk_cipher_encrypt,\
		.decrypt	= rk_cipher_decrypt,\
	} \
}

#define  RK_CIPHER_ALGO_XTS_INIT(cipher_algo, algo_name, driver_name) {\
	.name = #algo_name,\
	.type = ALG_TYPE_CIPHER,\
	.algo = CIPHER_ALGO_##cipher_algo,\
	.mode = CIPHER_MODE_XTS,\
	.alg.crypto = {\
		.base.cra_name		= #algo_name,\
		.base.cra_driver_name	= #driver_name,\
		.base.cra_priority	= RK_CRYPTO_PRIORITY,\
		.base.cra_flags		= CRYPTO_ALG_KERN_DRIVER_ONLY |\
					  CRYPTO_ALG_ASYNC |\
					  CRYPTO_ALG_NEED_FALLBACK,\
		.base.cra_blocksize	= cipher_algo##_BLOCK_SIZE,\
		.base.cra_ctxsize	= sizeof(struct rk_cipher_ctx),\
		.base.cra_alignmask	= 0x07,\
		.base.cra_module	= THIS_MODULE,\
		.init		= rk_ablk_init_tfm,\
		.exit		= rk_ablk_exit_tfm,\
		.min_keysize	= cipher_algo##_MAX_KEY_SIZE,\
		.max_keysize	= cipher_algo##_MAX_KEY_SIZE * 2,\
		.ivsize		= cipher_algo##_BLOCK_SIZE,\
		.chunksize      = cipher_algo##_BLOCK_SIZE,\
		.setkey		= rk_cipher_setkey,\
		.encrypt	= rk_cipher_encrypt,\
		.decrypt	= rk_cipher_decrypt,\
	} \
}

#define RK_HASH_ALGO_INIT(hash_algo, algo_name) {\
	.name = #algo_name,\
	.type = ALG_TYPE_HASH,\
	.algo = HASH_ALGO_##hash_algo,\
	.alg.hash = {\
		.init = rk_ahash_init,\
		.update = rk_ahash_update,\
		.final = rk_ahash_final,\
		.finup = rk_ahash_finup,\
		.export = rk_ahash_export,\
		.import = rk_ahash_import,\
		.digest = rk_ahash_digest,\
		.halg = {\
			.digestsize = hash_algo##_DIGEST_SIZE,\
			.statesize = sizeof(struct algo_name##_state),\
			.base = {\
				.cra_name = #algo_name,\
				.cra_driver_name = #algo_name"-rk",\
				.cra_priority = RK_CRYPTO_PRIORITY,\
				.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\
					     CRYPTO_ALG_ASYNC |\
					     CRYPTO_ALG_NEED_FALLBACK,\
				.cra_blocksize = hash_algo##_BLOCK_SIZE,\
				.cra_ctxsize = sizeof(struct rk_ahash_ctx),\
				.cra_alignmask = 0,\
				.cra_init = rk_cra_hash_init,\
				.cra_exit = rk_cra_hash_exit,\
				.cra_module = THIS_MODULE,\
			} \
		} \
	} \
}

#define RK_HMAC_ALGO_INIT(hash_algo, algo_name) {\
	.name = "hmac(" #algo_name ")",\
	.type = ALG_TYPE_HMAC,\
	.algo = HASH_ALGO_##hash_algo,\
	.alg.hash = {\
		.init = rk_ahash_init,\
		.update = rk_ahash_update,\
		.final = rk_ahash_final,\
		.finup = rk_ahash_finup,\
		.export = rk_ahash_export,\
		.import = rk_ahash_import,\
		.digest = rk_ahash_digest,\
		.setkey = rk_ahash_hmac_setkey,\
		.halg = {\
			.digestsize = hash_algo##_DIGEST_SIZE,\
			.statesize = sizeof(struct algo_name##_state),\
			.base = {\
				.cra_name = "hmac(" #algo_name ")",\
				.cra_driver_name = "hmac-" #algo_name "-rk",\
				.cra_priority = RK_CRYPTO_PRIORITY,\
				.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\
					     CRYPTO_ALG_ASYNC |\
					     CRYPTO_ALG_NEED_FALLBACK,\
				.cra_blocksize = hash_algo##_BLOCK_SIZE,\
				.cra_ctxsize = sizeof(struct rk_ahash_ctx),\
				.cra_alignmask = 0,\
				.cra_init = rk_cra_hash_init,\
				.cra_exit = rk_cra_hash_exit,\
				.cra_module = THIS_MODULE,\
			} \
		} \
	} \
}

#define IS_TYPE_HMAC(type) ((type) == ALG_TYPE_HMAC)

#define CRYPTO_READ(dev, offset)		  \
		readl_relaxed(((dev)->reg + (offset)))
#define CRYPTO_WRITE(dev, offset, val)	  \
		writel_relaxed((val), ((dev)->reg + (offset)))

#ifdef DEBUG
#define CRYPTO_TRACE(format, ...) pr_err("[%s, %05d]-trace: " format "\n", \
					 __func__, __LINE__, ##__VA_ARGS__)
#define CRYPTO_MSG(format, ...) pr_err("[%s, %05d]-msg:" format "\n", \
				       __func__, __LINE__, ##__VA_ARGS__)
#define CRYPTO_DUMPHEX(var_name, data, len) print_hex_dump(KERN_CONT, (var_name), \
							   DUMP_PREFIX_OFFSET, \
							   16, 1, (data), (len), false)
#else
#define CRYPTO_TRACE(format, ...)
#define CRYPTO_MSG(format, ...)
#define CRYPTO_DUMPHEX(var_name, data, len)
#endif

#endif