Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Memory copy functions for 32-bit PowerPC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 1996-2005 Paul Mackerras.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/ppc_asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/code-patching-asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #define COPY_16_BYTES		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	lwz	r7,4(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	lwz	r8,8(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	lwz	r9,12(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	lwzu	r10,16(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	stw	r7,4(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	stw	r8,8(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	stw	r9,12(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	stwu	r10,16(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define COPY_16_BYTES_WITHEX(n)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 8 ## n ## 0:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	lwz	r7,4(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 8 ## n ## 1:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	lwz	r8,8(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 8 ## n ## 2:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	lwz	r9,12(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 8 ## n ## 3:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	lwzu	r10,16(r4);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 8 ## n ## 4:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	stw	r7,4(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 8 ## n ## 5:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	stw	r8,8(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 8 ## n ## 6:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	stw	r9,12(r6);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 8 ## n ## 7:			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	stwu	r10,16(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #define COPY_16_BYTES_EXCODE(n)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 9 ## n ## 0:					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	addi	r5,r5,-(16 * n);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	b	104f;				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 9 ## n ## 1:					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	addi	r5,r5,-(16 * n);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	b	105f;				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	EX_TABLE(8 ## n ## 0b,9 ## n ## 0b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	EX_TABLE(8 ## n ## 1b,9 ## n ## 0b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	EX_TABLE(8 ## n ## 2b,9 ## n ## 0b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	EX_TABLE(8 ## n ## 3b,9 ## n ## 0b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	EX_TABLE(8 ## n ## 4b,9 ## n ## 1b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	EX_TABLE(8 ## n ## 5b,9 ## n ## 1b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	EX_TABLE(8 ## n ## 6b,9 ## n ## 1b);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	EX_TABLE(8 ## n ## 7b,9 ## n ## 1b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	.stabs	"arch/powerpc/lib/",N_SO,0,0,0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	.stabs	"copy_32.S",N_SO,0,0,0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) CACHELINE_BYTES = L1_CACHE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) LG_CACHELINE_BYTES = L1_CACHE_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) CACHELINE_MASK = (L1_CACHE_BYTES-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) #ifndef CONFIG_KASAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) _GLOBAL(memset16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	rlwinm.	r0 ,r5, 31, 1, 31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	addi	r6, r3, -4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	beq-	2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	rlwimi	r4 ,r4 ,16 ,0 ,15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 1:	stwu	r4, 4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	bdnz	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 2:	andi.	r0, r5, 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	sth	r4, 4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) EXPORT_SYMBOL(memset16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85)  * Use dcbz on the complete cache lines in the destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86)  * to set them to zero.  This requires that the destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87)  * area is cacheable.  -- paulus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89)  * During early init, cache might not be active yet, so dcbz cannot be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90)  * We therefore skip the optimised bloc that uses dcbz. This jump is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91)  * replaced by a nop once cache is active. This is done in machine_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) _GLOBAL_KASAN(memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	cmplwi	0,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	blt	7f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	rlwimi	r4,r4,8,16,23
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	rlwimi	r4,r4,16,0,15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	stw	r4,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	andi.	r0,r3,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	add	r5,r0,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	subf	r6,r0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	cmplwi	0,r4,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	 * Skip optimised bloc until cache is enabled. Will be replaced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	 * by 'bne' during boot to use normal procedure if r4 is not zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 5:	b	2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	patch_site	5b, patch__memset_nocache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	clrlwi	r7,r6,32-LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	add	r8,r7,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	srwi	r9,r8,LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	addic.	r9,r9,-1	/* total number of complete cachelines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	ble	2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	xori	r0,r7,CACHELINE_MASK & ~3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	srwi.	r0,r0,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	beq	3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 4:	stwu	r4,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	bdnz	4b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 3:	mtctr	r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	li	r7,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 10:	dcbz	r7,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	addi	r6,r6,CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	bdnz	10b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	clrlwi	r5,r8,32-LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	addi	r5,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 2:	srwi	r0,r5,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	bdz	6f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 1:	stwu	r4,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	bdnz	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 6:	andi.	r5,r5,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	mtctr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	addi	r6,r6,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 8:	stbu	r4,1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	bdnz	8b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 7:	cmpwi	0,r5,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	mtctr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	addi	r6,r3,-1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 9:	stbu	r4,1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	bdnz	9b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) EXPORT_SYMBOL(memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) EXPORT_SYMBOL_KASAN(memset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)  * This version uses dcbz on the complete cache lines in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)  * destination area to reduce memory traffic.  This requires that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)  * the destination area is cacheable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)  * We only use this version if the source and dest don't overlap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)  * -- paulus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)  * During early init, cache might not be active yet, so dcbz cannot be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)  * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)  * replaced by a nop once cache is active. This is done in machine_init()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) _GLOBAL_KASAN(memmove)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	cmplw	0,r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	bgt	backwards_memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	/* fall through */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) _GLOBAL_KASAN(memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 1:	b	generic_memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	patch_site	1b, patch__memcpy_nocache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	add	r7,r3,r5		/* test if the src & dst overlap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	add	r8,r4,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	cmplw	0,r4,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	cmplw	1,r3,r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	crand	0,0,4			/* cr0.lt &= cr1.lt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	blt	generic_memcpy		/* if regions overlap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	addi	r4,r4,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	addi	r6,r3,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	neg	r0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	andi.	r0,r0,CACHELINE_MASK	/* # bytes to start of cache line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	beq	58f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	cmplw	0,r5,r0			/* is this more than total to do? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	blt	63f			/* if not much to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	andi.	r8,r0,3			/* get it word-aligned first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	subf	r5,r0,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	mtctr	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	beq+	61f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 70:	lbz	r9,4(r4)		/* do some bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	addi	r6,r6,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	stb	r9,3(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	bdnz	70b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 61:	srwi.	r0,r0,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	beq	58f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 72:	lwzu	r9,4(r4)		/* do some words */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	stwu	r9,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	bdnz	72b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 58:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	li	r11,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	beq	63f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 53:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	dcbz	r11,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #if L1_CACHE_BYTES >= 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #if L1_CACHE_BYTES >= 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #if L1_CACHE_BYTES >= 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	COPY_16_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	bdnz	53b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 63:	srwi.	r0,r5,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	beq	64f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 30:	lwzu	r0,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	stwu	r0,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	bdnz	30b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 64:	andi.	r0,r5,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	beq+	65f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	addi	r4,r4,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	addi	r6,r6,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 40:	lbzu	r0,1(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	stbu	r0,1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	bdnz	40b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 65:	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) EXPORT_SYMBOL(memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) EXPORT_SYMBOL(memmove)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) EXPORT_SYMBOL_KASAN(memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) EXPORT_SYMBOL_KASAN(memmove)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) generic_memcpy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	srwi.	r7,r5,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	addi	r6,r3,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	addi	r4,r4,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	beq	2f			/* if less than 8 bytes to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	andi.	r0,r6,3			/* get dest word aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	bne	5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 1:	lwz	r7,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	lwzu	r8,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	stw	r7,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	stwu	r8,8(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	bdnz	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	andi.	r5,r5,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 2:	cmplwi	0,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	blt	3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	lwzu	r0,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	addi	r5,r5,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	stwu	r0,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 3:	cmpwi	0,r5,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	mtctr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	addi	r4,r4,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	addi	r6,r6,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 4:	lbzu	r0,1(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	stbu	r0,1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	bdnz	4b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 5:	subfic	r0,r0,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 6:	lbz	r7,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	stb	r7,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	addi	r6,r6,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	bdnz	6b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	subf	r5,r0,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	rlwinm.	r7,r5,32-3,3,31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	beq	2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	b	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) _GLOBAL(backwards_memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	rlwinm.	r7,r5,32-3,3,31		/* r0 = r5 >> 3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	add	r6,r3,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	add	r4,r4,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	beq	2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	andi.	r0,r6,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	bne	5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 1:	lwz	r7,-4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	lwzu	r8,-8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	stw	r7,-4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	stwu	r8,-8(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	bdnz	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	andi.	r5,r5,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 2:	cmplwi	0,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	blt	3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	lwzu	r0,-4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	subi	r5,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	stwu	r0,-4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 3:	cmpwi	0,r5,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	beqlr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	mtctr	r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 4:	lbzu	r0,-1(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	stbu	r0,-1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	bdnz	4b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 5:	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 6:	lbzu	r7,-1(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	stbu	r7,-1(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	bdnz	6b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	subf	r5,r0,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	rlwinm.	r7,r5,32-3,3,31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	beq	2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	b	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) _GLOBAL(__copy_tofrom_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	addi	r4,r4,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	addi	r6,r3,-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	neg	r0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	andi.	r0,r0,CACHELINE_MASK	/* # bytes to start of cache line */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	beq	58f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	cmplw	0,r5,r0			/* is this more than total to do? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	blt	63f			/* if not much to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	andi.	r8,r0,3			/* get it word-aligned first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	mtctr	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	beq+	61f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 70:	lbz	r9,4(r4)		/* do some bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 71:	stb	r9,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	addi	r6,r6,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	bdnz	70b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 61:	subf	r5,r0,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	srwi.	r0,r0,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	beq	58f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 72:	lwzu	r9,4(r4)		/* do some words */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 73:	stwu	r9,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	bdnz	72b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	EX_TABLE(70b,100f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	EX_TABLE(71b,101f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	EX_TABLE(72b,102f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	EX_TABLE(73b,103f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 58:	srwi.	r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	clrlwi	r5,r5,32-LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	li	r11,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	beq	63f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	/* Here we decide how far ahead to prefetch the source */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	li	r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	cmpwi	r0,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	li	r7,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	ble	114f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	li	r7,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #if MAX_COPY_PREFETCH > 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	/* Heuristically, for large transfers we prefetch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	   MAX_COPY_PREFETCH cachelines ahead.  For small transfers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	   we prefetch 1 cacheline ahead. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	cmpwi	r0,MAX_COPY_PREFETCH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	ble	112f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	li	r7,MAX_COPY_PREFETCH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 112:	mtctr	r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 111:	dcbt	r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	addi	r3,r3,CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	bdnz	111b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	dcbt	r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 	addi	r3,r3,CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) #endif /* MAX_COPY_PREFETCH > 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 114:	subf	r8,r7,r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	mr	r0,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	mtctr	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 53:	dcbt	r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 54:	dcbz	r11,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	EX_TABLE(54b,105f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* the main body of the cacheline loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	COPY_16_BYTES_WITHEX(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) #if L1_CACHE_BYTES >= 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	COPY_16_BYTES_WITHEX(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) #if L1_CACHE_BYTES >= 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	COPY_16_BYTES_WITHEX(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	COPY_16_BYTES_WITHEX(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) #if L1_CACHE_BYTES >= 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	COPY_16_BYTES_WITHEX(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 	COPY_16_BYTES_WITHEX(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	COPY_16_BYTES_WITHEX(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 	COPY_16_BYTES_WITHEX(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 	bdnz	53b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	cmpwi	r0,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	li	r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	li	r7,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	bne	114b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 63:	srwi.	r0,r5,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	beq	64f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 30:	lwzu	r0,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 31:	stwu	r0,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	bdnz	30b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 64:	andi.	r0,r5,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	mtctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	beq+	65f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 40:	lbz	r0,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 41:	stb	r0,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	addi	r6,r6,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 	bdnz	40b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 65:	li	r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /* read fault, initial single-byte copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 100:	li	r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 	b	90f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) /* write fault, initial single-byte copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 101:	li	r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 90:	subf	r5,r8,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 	li	r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	b	99f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* read fault, initial word copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 102:	li	r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 	b	91f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) /* write fault, initial word copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 103:	li	r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 91:	li	r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	b	99f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)  * this stuff handles faults in the cacheline loop and branches to either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)  * 104f (if in read part) or 105f (if in write part), after updating r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	COPY_16_BYTES_EXCODE(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) #if L1_CACHE_BYTES >= 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 	COPY_16_BYTES_EXCODE(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) #if L1_CACHE_BYTES >= 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	COPY_16_BYTES_EXCODE(2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 	COPY_16_BYTES_EXCODE(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) #if L1_CACHE_BYTES >= 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	COPY_16_BYTES_EXCODE(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	COPY_16_BYTES_EXCODE(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 	COPY_16_BYTES_EXCODE(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	COPY_16_BYTES_EXCODE(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /* read fault in cacheline loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 104:	li	r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	b	92f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) /* fault on dcbz (effectively a write fault) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* or write fault in cacheline loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 105:	li	r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 92:	li	r3,LG_CACHELINE_BYTES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	mfctr	r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 	add	r0,r0,r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	b	106f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /* read fault in final word loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 108:	li	r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 	b	93f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /* write fault in final word loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 109:	li	r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 93:	andi.	r5,r5,3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	li	r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	b	99f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /* read fault in final byte loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 110:	li	r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 	b	94f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /* write fault in final byte loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 111:	li	r9,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 94:	li	r5,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 	li	r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)  * At this stage the number of bytes not copied is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)  * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 99:	mfctr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 106:	slw	r3,r0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 	add.	r3,r3,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	beq	120f			/* shouldn't happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	cmpwi	0,r9,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	bne	120f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /* for a read fault, first try to continue the copy one byte at a time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 	mtctr	r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 130:	lbz	r0,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 131:	stb	r0,4(r6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	addi	r6,r6,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	bdnz	130b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* then clear out the destination: r3 bytes starting at 4(r6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) 132:	mfctr	r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 120:	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 	EX_TABLE(30b,108b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	EX_TABLE(31b,109b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 	EX_TABLE(40b,110b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 	EX_TABLE(41b,111b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) 	EX_TABLE(130b,132b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 	EX_TABLE(131b,120b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) EXPORT_SYMBOL(__copy_tofrom_user)