Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Crypto-API module for CRC-32 algorithms implemented with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * z/Architecture Vector Extension Facility.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright IBM Corp. 2015
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #define KMSG_COMPONENT	"crc32-vx"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/crc32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <crypto/internal/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <asm/fpu/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define CRC32_BLOCK_SIZE	1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define CRC32_DIGEST_SIZE	4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define VX_MIN_LEN		64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #define VX_ALIGNMENT		16L
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define VX_ALIGN_MASK		(VX_ALIGNMENT - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) struct crc_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	u32 key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) struct crc_desc_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	u32 crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) /* Prototypes for functions in assembly files */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  * Creates a function to perform a particular CRC-32 computation. Depending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  * on the message buffer, the hardware-accelerated or software implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  * is used.   Note that the message buffer is aligned to improve fetch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * operations of VECTOR LOAD MULTIPLE instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	static u32 __pure ___fname(u32 crc,				    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 				unsigned char const *data, size_t datalen)  \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	{								    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		struct kernel_fpu vxstate;				    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 		unsigned long prealign, aligned, remaining;		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		if (datalen < VX_MIN_LEN + VX_ALIGN_MASK)		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 			return ___crc32_sw(crc, data, datalen);		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		if ((unsigned long)data & VX_ALIGN_MASK) {		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 			prealign = VX_ALIGNMENT -			    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 				  ((unsigned long)data & VX_ALIGN_MASK);    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 			datalen -= prealign;				    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 			crc = ___crc32_sw(crc, data, prealign);		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 			data = (void *)((unsigned long)data + prealign);    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 		}							    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		aligned = datalen & ~VX_ALIGN_MASK;			    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 		remaining = datalen & VX_ALIGN_MASK;			    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		crc = ___crc32_vx(crc, data, aligned);			    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);		    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 		if (remaining)						    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 			crc = ___crc32_sw(crc, data + aligned, remaining);  \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 									    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 		return crc;						    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	mctx->key = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	mctx->key = ~0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static int crc32_vx_init(struct shash_desc *desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	ctx->crc = mctx->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 			   unsigned int newkeylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	struct crc_ctx *mctx = crypto_shash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	if (newkeylen != sizeof(mctx->key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	mctx->key = le32_to_cpu(*(__le32 *)newkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 			     unsigned int newkeylen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct crc_ctx *mctx = crypto_shash_ctx(tfm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	if (newkeylen != sizeof(mctx->key))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	mctx->key = be32_to_cpu(*(__be32 *)newkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	*(__le32 *)out = cpu_to_le32p(&ctx->crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	*(__be32 *)out = cpu_to_be32p(&ctx->crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	 * Perform a final XOR with 0xFFFFFFFF to be in sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	 * with the generic crc32c shash implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	*(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			      u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	*(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 			      u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	*(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 			     u8 *out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	 * Perform a final XOR with 0xFFFFFFFF to be in sync
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	 * with the generic crc32c shash implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	*(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define CRC32_VX_FINUP(alg, func)					      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data,  \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 				   unsigned int datalen, u8 *out)	      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	{								      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		return __ ## alg ## _vx_finup(shash_desc_ctx(desc),	      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 					      data, datalen, out);	      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) CRC32_VX_FINUP(crc32le, crc32_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) CRC32_VX_FINUP(crc32be, crc32_be_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) CRC32_VX_FINUP(crc32c, crc32c_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #define CRC32_VX_DIGEST(alg, func)					      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 				     unsigned int len, u8 *out)		      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	{								      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm),    \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 					      data, len, out);		      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) CRC32_VX_DIGEST(crc32le, crc32_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) CRC32_VX_DIGEST(crc32be, crc32_be_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #define CRC32_VX_UPDATE(alg, func)					      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 				     unsigned int datalen)		      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	{								      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		struct crc_desc_ctx *ctx = shash_desc_ctx(desc);	      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		ctx->crc = func(ctx->crc, data, datalen);		      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		return 0;						      \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) CRC32_VX_UPDATE(crc32le, crc32_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) CRC32_VX_UPDATE(crc32be, crc32_be_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static struct shash_alg crc32_vx_algs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	/* CRC-32 LE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		.init		=	crc32_vx_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		.setkey		=	crc32_vx_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		.update		=	crc32le_vx_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		.final		=	crc32le_vx_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		.finup		=	crc32le_vx_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		.digest		=	crc32le_vx_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 		.descsize	=	sizeof(struct crc_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		.digestsize	=	CRC32_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		.base		=	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			.cra_name	 = "crc32",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 			.cra_driver_name = "crc32-vx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 			.cra_priority	 = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 			.cra_ctxsize	 = sizeof(struct crc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			.cra_module	 = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 			.cra_init	 = crc32_vx_cra_init_zero,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	/* CRC-32 BE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 		.init		=	crc32_vx_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 		.setkey		=	crc32be_vx_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 		.update		=	crc32be_vx_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 		.final		=	crc32be_vx_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 		.finup		=	crc32be_vx_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		.digest		=	crc32be_vx_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 		.descsize	=	sizeof(struct crc_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		.digestsize	=	CRC32_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		.base		=	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 			.cra_name	 = "crc32be",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 			.cra_driver_name = "crc32be-vx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 			.cra_priority	 = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 			.cra_ctxsize	 = sizeof(struct crc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 			.cra_module	 = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 			.cra_init	 = crc32_vx_cra_init_zero,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	/* CRC-32C LE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		.init		=	crc32_vx_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		.setkey		=	crc32_vx_setkey,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		.update		=	crc32c_vx_update,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 		.final		=	crc32c_vx_final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		.finup		=	crc32c_vx_finup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 		.digest		=	crc32c_vx_digest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 		.descsize	=	sizeof(struct crc_desc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		.digestsize	=	CRC32_DIGEST_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 		.base		=	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 			.cra_name	 = "crc32c",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 			.cra_driver_name = "crc32c-vx",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 			.cra_priority	 = 200,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 			.cra_ctxsize	 = sizeof(struct crc_ctx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 			.cra_module	 = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 			.cra_init	 = crc32_vx_cra_init_invert,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 		},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static int __init crc_vx_mod_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	return crypto_register_shashes(crc32_vx_algs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 				       ARRAY_SIZE(crc32_vx_algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static void __exit crc_vx_mod_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) module_cpu_feature_match(VXRS, crc_vx_mod_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) module_exit(crc_vx_mod_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) MODULE_ALIAS_CRYPTO("crc32");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) MODULE_ALIAS_CRYPTO("crc32-vx");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) MODULE_ALIAS_CRYPTO("crc32c");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) MODULE_ALIAS_CRYPTO("crc32c-vx");