Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * License.  See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Quick'n'dirty IP checksum ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * Copyright (C) 1998, 1999 Ralf Baechle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * Copyright (C) 1999 Silicon Graphics, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * Copyright (C) 2007  Maciej W. Rozycki
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * Copyright (C) 2014 Imagination Technologies Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <asm/regdef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  * As we are sharing code base with the mips32 tree (which use the o32 ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * register definitions). We need to redefine the register definitions from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * the n64 ABI register naming to the o32 ABI register naming.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #undef t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #undef t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #undef t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #undef t3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #define t0	$8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #define t1	$9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #define t2	$10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #define t3	$11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #define t4	$12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #define t5	$13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #define t6	$14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #define t7	$15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #define USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #define LOAD   ld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #define LOAD32 lwu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #define ADD    daddu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #define NBYTES 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #define LOAD   lw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #define LOAD32 lw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #define ADD    addu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #define NBYTES 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #endif /* USE_DOUBLE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) #define UNIT(unit)  ((unit)*NBYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) #define ADDC(sum,reg)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	.set	push;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	.set	noat;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	ADD	sum, reg;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	sltu	v1, sum, reg;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	ADD	sum, v1;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	.set	pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) #define ADDC32(sum,reg)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	.set	push;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	.set	noat;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	addu	sum, reg;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	sltu	v1, sum, reg;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	addu	sum, v1;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	.set	pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	LOAD	_t0, (offset + UNIT(0))(src);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	LOAD	_t1, (offset + UNIT(1))(src);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	LOAD	_t2, (offset + UNIT(2))(src);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	LOAD	_t3, (offset + UNIT(3))(src);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	ADDC(_t0, _t1);						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	ADDC(_t2, _t3);						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	ADDC(sum, _t0);						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	ADDC(sum, _t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95)  * a0: source address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96)  * a1: length of the area to checksum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97)  * a2: partial checksum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define src a0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define sum v0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	.align	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) LEAF(csum_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) EXPORT_SYMBOL(csum_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	move	sum, zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	move	t7, zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	sltiu	t8, a1, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	bnez	t8, .Lsmall_csumcpy		/* < 8 bytes to copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	 move	t2, a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	andi	t7, src, 0x1			/* odd buffer? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) .Lhword_align:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	beqz	t7, .Lword_align
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	 andi	t8, src, 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	lbu	t0, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	LONG_SUBU	a1, a1, 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #ifdef __MIPSEL__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	sll	t0, t0, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	PTR_ADDU	src, src, 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	andi	t8, src, 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) .Lword_align:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	beqz	t8, .Ldword_align
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	 sltiu	t8, a1, 56
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	lhu	t0, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	LONG_SUBU	a1, a1, 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	sltiu	t8, a1, 56
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	PTR_ADDU	src, src, 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) .Ldword_align:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	bnez	t8, .Ldo_end_words
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	 move	t8, a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	andi	t8, src, 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	beqz	t8, .Lqword_align
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	 andi	t8, src, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	LOAD32	t0, 0x00(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	LONG_SUBU	a1, a1, 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	PTR_ADDU	src, src, 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	andi	t8, src, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) .Lqword_align:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	beqz	t8, .Loword_align
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	 andi	t8, src, 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	ld	t0, 0x00(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	LONG_SUBU	a1, a1, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	lw	t0, 0x00(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	lw	t1, 0x04(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	LONG_SUBU	a1, a1, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	ADDC(sum, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	PTR_ADDU	src, src, 0x8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	andi	t8, src, 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) .Loword_align:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	beqz	t8, .Lbegin_movement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	 LONG_SRL	t8, a1, 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	ld	t0, 0x00(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	ld	t1, 0x08(src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	ADDC(sum, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	LONG_SUBU	a1, a1, 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	PTR_ADDU	src, src, 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	LONG_SRL	t8, a1, 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) .Lbegin_movement:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	beqz	t8, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	 andi	t2, a1, 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) .Lmove_128bytes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	LONG_SUBU	t8, t8, 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	PTR_ADDU	src, src, 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	bnez	t8, .Lmove_128bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	beqz	t2, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	 andi	t2, a1, 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) .Lmove_64bytes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	PTR_ADDU	src, src, 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	beqz	t2, .Ldo_end_words
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	 andi	t8, a1, 0x1c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) .Lmove_32bytes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	andi	t8, a1, 0x1c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	PTR_ADDU	src, src, 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) .Ldo_end_words:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	beqz	t8, .Lsmall_csumcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	 andi	t2, a1, 0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	LONG_SRL	t8, t8, 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) .Lend_words:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	LOAD32	t0, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	LONG_SUBU	t8, t8, 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	PTR_ADDU	src, src, 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	bnez	t8, .Lend_words
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) /* unknown src alignment and < 8 bytes to go  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) .Lsmall_csumcpy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	move	a1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	andi	t0, a1, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	beqz	t0, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	 andi	t0, a1, 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	/* Still a full word to go  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	ulw	t1, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	PTR_ADDIU	src, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	dsll	t1, t1, 32			/* clear lower 32bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	ADDC(sum, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 1:	move	t1, zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	beqz	t0, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	 andi	t0, a1, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	/* Still a halfword to go  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	ulhu	t1, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	PTR_ADDIU	src, 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 1:	beqz	t0, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	 sll	t1, t1, 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	lbu	t2, (src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #ifdef __MIPSEB__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	sll	t2, t2, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	or	t1, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 1:	ADDC(sum, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	/* fold checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	dsll32	v1, sum, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	daddu	sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	sltu	v1, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	dsra32	sum, sum, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	addu	sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	/* odd buffer alignment? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)     defined(CONFIG_CPU_LOONGSON64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	.set	push
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	.set	arch=mips32r2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	wsbh	v1, sum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	movn	sum, v1, t7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	.set	pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	beqz	t7, 1f			/* odd buffer alignment? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	 lui	v1, 0x00ff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	addu	v1, 0x00ff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	and	t0, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	sll	t0, t0, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	srl	sum, sum, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	and	sum, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	or	sum, sum, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	.set	reorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	/* Add the passed partial csum.	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	ADDC32(sum, a2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	jr	ra
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	END(csum_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)  * checksum and copy routines based on memcpy.S
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)  *	csum_partial_copy_nocheck(src, dst, len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)  *	__csum_partial_copy_kernel(src, dst, len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)  * See "Spec" in memcpy.S for details.	Unlike __copy_user, all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)  * function in this file use the standard calling convention.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) #define src a0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) #define dst a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) #define len a2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #define sum v0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) #define odd t8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)  * All exception handlers simply return 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* Instruction type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) #define LD_INSN 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #define ST_INSN 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) #define LEGACY_MODE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #define EVA_MODE    2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) #define USEROP   1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) #define KERNELOP 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)  * Wrapper to add an entry in the exception table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)  * in case the insn causes a memory exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)  * Arguments:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)  * insn    : Load/store instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)  * type    : Instruction type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)  * reg     : Register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)  * addr    : Address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)  * handler : Exception handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #define EXC(insn, type, reg, addr)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	.if \mode == LEGACY_MODE;		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 9:		insn reg, addr;			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		.section __ex_table,"a";	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 		PTR	9b, .L_exc;		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 		.previous;			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	/* This is enabled in EVA mode */	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	.else;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 		/* If loading from user or storing to user */	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 		.if ((\from == USEROP) && (type == LD_INSN)) || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		    ((\to == USEROP) && (type == ST_INSN));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 9:			__BUILD_EVA_INSN(insn##e, reg, addr);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 			.section __ex_table,"a";		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 			PTR	9b, .L_exc;			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 			.previous;				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 		.else;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 			/* EVA without exception */		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 			insn reg, addr;				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 		.endif;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	.endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #undef LOAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) #define LOADK	ld /* No exception */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) #define LOAD(reg, addr)		EXC(ld, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) #define LOADBU(reg, addr)	EXC(lbu, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) #define LOADL(reg, addr)	EXC(ldl, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) #define LOADR(reg, addr)	EXC(ldr, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) #define STOREB(reg, addr)	EXC(sb, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) #define STOREL(reg, addr)	EXC(sdl, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) #define STORER(reg, addr)	EXC(sdr, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) #define STORE(reg, addr)	EXC(sd, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #define ADD    daddu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) #define SUB    dsubu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) #define SRL    dsrl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #define SLL    dsll
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #define SLLV   dsllv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #define SRLV   dsrlv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #define NBYTES 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) #define LOG_NBYTES 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #define LOADK	lw /* No exception */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) #define LOAD(reg, addr)		EXC(lw, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) #define LOADBU(reg, addr)	EXC(lbu, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) #define LOADL(reg, addr)	EXC(lwl, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) #define LOADR(reg, addr)	EXC(lwr, LD_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #define STOREB(reg, addr)	EXC(sb, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) #define STOREL(reg, addr)	EXC(swl, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) #define STORER(reg, addr)	EXC(swr, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) #define STORE(reg, addr)	EXC(sw, ST_INSN, reg, addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) #define ADD    addu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) #define SUB    subu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) #define SRL    srl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #define SLL    sll
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #define SLLV   sllv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #define SRLV   srlv
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) #define NBYTES 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) #define LOG_NBYTES 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) #endif /* USE_DOUBLE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) #ifdef CONFIG_CPU_LITTLE_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) #define LDFIRST LOADR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #define LDREST	LOADL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) #define STFIRST STORER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) #define STREST	STOREL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) #define SHIFT_DISCARD SLLV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #define SHIFT_DISCARD_REVERT SRLV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) #define LDFIRST LOADL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) #define LDREST	LOADR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) #define STFIRST STOREL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) #define STREST	STORER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) #define SHIFT_DISCARD SRLV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #define SHIFT_DISCARD_REVERT SLLV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #define FIRST(unit) ((unit)*NBYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) #define REST(unit)  (FIRST(unit)+NBYTES-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #define ADDRMASK (NBYTES-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 	.set	noat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	.set	at=v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 	.macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 	li	sum, -1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	move	odd, zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 	 * Note: dst & src may be unaligned, len may be 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	 * Temps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 	 * The "issue break"s below are very approximate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	 * Issue delays for dcache fills will perturb the schedule, as will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	 * load queue full replay traps, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	 * If len < NBYTES use byte operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	sltu	t2, len, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	and	t1, dst, ADDRMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 	bnez	t2, .Lcopy_bytes_checklen\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 	 and	t0, src, ADDRMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	andi	odd, dst, 0x1			/* odd buffer? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	bnez	t1, .Ldst_unaligned\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	bnez	t0, .Lsrc_unaligned_dst_aligned\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	 * use delay slot for fall-through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 	 * src and dst are aligned; need to compute rem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) .Lboth_aligned\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 	 SRL	t0, len, LOG_NBYTES+3	 # +3 for 8 units/iter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	beqz	t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	SUB	len, 8*NBYTES		# subtract here for bgez loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	.align	4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	LOAD(t0, UNIT(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 	LOAD(t1, UNIT(1)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	LOAD(t2, UNIT(2)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 	LOAD(t3, UNIT(3)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 	LOAD(t4, UNIT(4)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 	LOAD(t5, UNIT(5)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 	LOAD(t6, UNIT(6)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 	LOAD(t7, UNIT(7)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 	SUB	len, len, 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	ADD	src, src, 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	STORE(t0, UNIT(0)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 	ADDC(t0, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 	STORE(t1, UNIT(1)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 	STORE(t2, UNIT(2)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 	ADDC(t2, t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 	STORE(t3, UNIT(3)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 	ADDC(sum, t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	STORE(t4, UNIT(4)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 	ADDC(t4, t5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 	STORE(t5, UNIT(5)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 	ADDC(sum, t4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 	STORE(t6, UNIT(6)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	ADDC(t6, t7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 	STORE(t7, UNIT(7)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	ADDC(sum, t6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	ADD	dst, dst, 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 	bgez	len, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 	ADD	len, 8*NBYTES		# revert len (see above)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	 * len == the number of bytes left to copy < 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) .Lcleanup_both_aligned\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) #define rem t7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 	beqz	len, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 	 sltu	t0, len, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 	bnez	t0, .Lless_than_4units\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	 and	rem, len, (NBYTES-1)	# rem = len % NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 	 * len >= 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 	LOAD(t0, UNIT(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 	LOAD(t1, UNIT(1)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 	LOAD(t2, UNIT(2)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) 	LOAD(t3, UNIT(3)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) 	SUB	len, len, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) 	ADD	src, src, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) 	STORE(t0, UNIT(0)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 	ADDC(t0, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) 	STORE(t1, UNIT(1)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) 	STORE(t2, UNIT(2)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) 	ADDC(t2, t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 	STORE(t3, UNIT(3)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) 	ADDC(sum, t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 	ADD	dst, dst, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 	beqz	len, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) .Lless_than_4units\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) 	 * rem = len % NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) 	beq	rem, len, .Lcopy_bytes\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) 	LOAD(t0, 0(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) 	ADD	src, src, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) 	SUB	len, len, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) 	STORE(t0, 0(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) 	ADD	dst, dst, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) 	bne	rem, len, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) 	 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) 	 * A loop would do only a byte at a time with possible branch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) 	 * mispredicts.	 Can't do an explicit LOAD dst,mask,or,STORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) 	 * because can't assume read-access to dst.  Instead, use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) 	 * STREST dst, which doesn't require read access to dst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) 	 * This code should perform better than a simple loop on modern,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) 	 * wide-issue mips processors because the code has fewer branches and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) 	 * more instruction-level parallelism.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) #define bits t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 	beqz	len, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) 	 ADD	t1, dst, len	# t1 is just past last byte of dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) 	li	bits, 8*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) 	SLL	rem, len, 3	# rem = number of bits to keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 	LOAD(t0, 0(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) 	SUB	bits, bits, rem # bits = number of bits to discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) 	SHIFT_DISCARD t0, t0, bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) 	STREST(t0, -1(t1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) 	SHIFT_DISCARD_REVERT t0, t0, bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) 	.set reorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) 	b	.Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) 	.set noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) .Ldst_unaligned\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) 	 * dst is unaligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) 	 * t0 = src & ADDRMASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) 	 * t1 = dst & ADDRMASK; T1 > 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) 	 * len >= NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) 	 * Copy enough bytes to align dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) 	 * Set match = (src and dst have same alignment)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) #define match rem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) 	LDFIRST(t3, FIRST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) 	ADD	t2, zero, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) 	LDREST(t3, REST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) 	SUB	t2, t2, t1	# t2 = number of bytes copied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) 	xor	match, t0, t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) 	STFIRST(t3, FIRST(0)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) 	SLL	t4, t1, 3		# t4 = number of bits to discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) 	SHIFT_DISCARD t3, t3, t4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) 	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) 	ADDC(sum, t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) 	beq	len, t2, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) 	 SUB	len, len, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) 	ADD	dst, dst, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) 	beqz	match, .Lboth_aligned\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) 	 ADD	src, src, t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) .Lsrc_unaligned_dst_aligned\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) 	SRL	t0, len, LOG_NBYTES+2	 # +2 for 4 units/iter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) 	beqz	t0, .Lcleanup_src_unaligned\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) 	 and	rem, len, (4*NBYTES-1)	 # rem = len % 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)  * Avoid consecutive LD*'s to the same register since some mips
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)  * implementations can't issue them in the same cycle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)  * It's OK to load FIRST(N+1) before REST(N) because the two addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)  * are to the same unit (unless src is aligned, but it's not).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) 	LDFIRST(t0, FIRST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) 	LDFIRST(t1, FIRST(1)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) 	SUB	len, len, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) 	LDREST(t0, REST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) 	LDREST(t1, REST(1)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) 	LDFIRST(t2, FIRST(2)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) 	LDFIRST(t3, FIRST(3)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) 	LDREST(t2, REST(2)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) 	LDREST(t3, REST(3)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) 	ADD	src, src, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) #ifdef CONFIG_CPU_SB1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) 	nop				# improves slotting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) 	STORE(t0, UNIT(0)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) 	ADDC(t0, t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) 	STORE(t1, UNIT(1)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) 	STORE(t2, UNIT(2)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) 	ADDC(t2, t3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) 	STORE(t3, UNIT(3)(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) 	ADDC(sum, t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) 	ADD	dst, dst, 4*NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) 	bne	len, rem, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) .Lcleanup_src_unaligned\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) 	beqz	len, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) 	 and	rem, len, NBYTES-1  # rem = len % NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) 	beq	rem, len, .Lcopy_bytes\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) 	LDFIRST(t0, FIRST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) 	LDREST(t0, REST(0)(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) 	ADD	src, src, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) 	SUB	len, len, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) 	STORE(t0, 0(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) 	ADDC(sum, t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) 	.set	reorder				/* DADDI_WAR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) 	ADD	dst, dst, NBYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) 	bne	len, rem, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) 	.set	noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) .Lcopy_bytes_checklen\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) 	beqz	len, .Ldone\@
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) 	 nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) .Lcopy_bytes\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) 	/* 0 < len < NBYTES  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) #ifdef CONFIG_CPU_LITTLE_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) #define SHIFT_START 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) #define SHIFT_INC 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) #define SHIFT_START 8*(NBYTES-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) #define SHIFT_INC -8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) 	move	t2, zero	# partial word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) 	li	t3, SHIFT_START # shift
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) #define COPY_BYTE(N)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) 	LOADBU(t0, N(src));		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) 	SUB	len, len, 1;		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) 	STOREB(t0, N(dst));		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) 	SLLV	t0, t0, t3;		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) 	addu	t3, SHIFT_INC;		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) 	beqz	len, .Lcopy_bytes_done\@; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) 	 or	t2, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) 	COPY_BYTE(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) 	COPY_BYTE(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) 	COPY_BYTE(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) 	COPY_BYTE(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) 	COPY_BYTE(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) 	COPY_BYTE(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) 	LOADBU(t0, NBYTES-2(src))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) 	SUB	len, len, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) 	STOREB(t0, NBYTES-2(dst))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) 	SLLV	t0, t0, t3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) 	or	t2, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) .Lcopy_bytes_done\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) 	ADDC(sum, t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) .Ldone\@:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) 	/* fold checksum */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) 	.set	push
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) 	.set	noat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) #ifdef USE_DOUBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) 	dsll32	v1, sum, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) 	daddu	sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) 	sltu	v1, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) 	dsra32	sum, sum, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) 	addu	sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)     defined(CONFIG_CPU_LOONGSON64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) 	.set	push
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) 	.set	arch=mips32r2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) 	wsbh	v1, sum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) 	movn	sum, v1, odd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) 	.set	pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) 	beqz	odd, 1f			/* odd buffer alignment? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) 	 lui	v1, 0x00ff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) 	addu	v1, 0x00ff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) 	and	t0, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) 	sll	t0, t0, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) 	srl	sum, sum, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) 	and	sum, sum, v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) 	or	sum, sum, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) 	.set	pop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) 	.set reorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) 	jr	ra
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) 	.set noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) 	.set noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) .L_exc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) 	jr	ra
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) 	 li	v0, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) FEXPORT(__csum_partial_copy_nocheck)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) EXPORT_SYMBOL(__csum_partial_copy_nocheck)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) #ifndef CONFIG_EVA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) FEXPORT(__csum_partial_copy_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) EXPORT_SYMBOL(__csum_partial_copy_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) FEXPORT(__csum_partial_copy_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) EXPORT_SYMBOL(__csum_partial_copy_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) __BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) #ifdef CONFIG_EVA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) LEAF(__csum_partial_copy_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) END(__csum_partial_copy_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) LEAF(__csum_partial_copy_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) __BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) END(__csum_partial_copy_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) #endif