Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Copyright (C) IBM Corporation, 2011
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Author: Anton Blanchard <anton@au.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/ppc_asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #ifndef SELFTEST_CASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) /* 0 == don't use VMX, 1 == use VMX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #define SELFTEST_CASE	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #ifdef __BIG_ENDIAN__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #define LVS(VRT,RA,RB)		lvsl	VRT,RA,RB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRA,VRB,VRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define LVS(VRT,RA,RB)		lvsr	VRT,RA,RB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRB,VRA,VRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	.macro err1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 100:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	EX_TABLE(100b,.Ldo_err1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	.macro err2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 200:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	EX_TABLE(200b,.Ldo_err2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #ifdef CONFIG_ALTIVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	.macro err3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 300:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	EX_TABLE(300b,.Ldo_err3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	.macro err4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 400:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	EX_TABLE(400b,.Ldo_err4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) .Ldo_err4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	ld	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	ld	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	ld	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) .Ldo_err3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	bl	exit_vmx_usercopy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	ld	r0,STACKFRAMESIZE+16(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	b	.Lexit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #endif /* CONFIG_ALTIVEC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) .Ldo_err2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	ld	r22,STK_REG(R22)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	ld	r21,STK_REG(R21)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	ld	r20,STK_REG(R20)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	ld	r19,STK_REG(R19)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	ld	r18,STK_REG(R18)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	ld	r17,STK_REG(R17)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	ld	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	ld	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	ld	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) .Lexit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	addi	r1,r1,STACKFRAMESIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) .Ldo_err1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	ld	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	ld	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	b	__copy_tofrom_user_base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) _GLOBAL(__copy_tofrom_user_power7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	cmpldi	r5,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	cmpldi	cr1,r5,3328
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	std	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	blt	.Lshort_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) #ifdef CONFIG_ALTIVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) test_feature = SELFTEST_CASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) BEGIN_FTR_SECTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	bgt	cr1,.Lvmx_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) .Lnonvmx_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	/* Get the source 8B aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	neg	r6,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	clrldi	r6,r6,(64-3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	bf	cr7*4+3,1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) err1;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) err1;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	addi	r3,r3,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 1:	bf	cr7*4+2,2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) err1;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) err1;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 2:	bf	cr7*4+1,3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) err1;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) err1;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 3:	sub	r5,r5,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	cmpldi	r5,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	blt	5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	stdu	r1,-STACKFRAMESIZE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	std	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	std	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	std	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	std	r17,STK_REG(R17)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	std	r18,STK_REG(R18)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	std	r19,STK_REG(R19)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	std	r20,STK_REG(R20)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	std	r21,STK_REG(R21)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	std	r22,STK_REG(R22)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	std	r0,STACKFRAMESIZE+16(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	srdi	r6,r5,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	/* Now do cacheline (128B) sized loads and stores. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	.align	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) err2;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) err2;	ld	r6,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) err2;	ld	r7,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) err2;	ld	r8,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) err2;	ld	r9,32(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) err2;	ld	r10,40(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) err2;	ld	r11,48(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) err2;	ld	r12,56(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) err2;	ld	r14,64(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) err2;	ld	r15,72(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) err2;	ld	r16,80(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) err2;	ld	r17,88(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) err2;	ld	r18,96(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) err2;	ld	r19,104(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) err2;	ld	r20,112(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) err2;	ld	r21,120(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	addi	r4,r4,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) err2;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) err2;	std	r6,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) err2;	std	r7,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) err2;	std	r8,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) err2;	std	r9,32(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) err2;	std	r10,40(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) err2;	std	r11,48(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) err2;	std	r12,56(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) err2;	std	r14,64(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) err2;	std	r15,72(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) err2;	std	r16,80(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) err2;	std	r17,88(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) err2;	std	r18,96(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) err2;	std	r19,104(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) err2;	std	r20,112(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) err2;	std	r21,120(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	addi	r3,r3,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	bdnz	4b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	clrldi	r5,r5,(64-7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	ld	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	ld	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	ld	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	ld	r17,STK_REG(R17)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	ld	r18,STK_REG(R18)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	ld	r19,STK_REG(R19)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	ld	r20,STK_REG(R20)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	ld	r21,STK_REG(R21)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	ld	r22,STK_REG(R22)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	addi	r1,r1,STACKFRAMESIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	/* Up to 127B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 5:	srdi	r6,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 6:	bf	cr7*4+1,7f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) err1;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) err1;	ld	r6,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) err1;	ld	r7,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) err1;	ld	r8,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) err1;	ld	r9,32(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) err1;	ld	r10,40(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) err1;	ld	r11,48(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) err1;	ld	r12,56(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	addi	r4,r4,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) err1;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) err1;	std	r6,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) err1;	std	r7,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) err1;	std	r8,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) err1;	std	r9,32(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) err1;	std	r10,40(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) err1;	std	r11,48(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) err1;	std	r12,56(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	addi	r3,r3,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	/* Up to 63B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 7:	bf	cr7*4+2,8f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) err1;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) err1;	ld	r6,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) err1;	ld	r7,16(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) err1;	ld	r8,24(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	addi	r4,r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) err1;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) err1;	std	r6,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) err1;	std	r7,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) err1;	std	r8,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	/* Up to 31B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 8:	bf	cr7*4+3,9f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) err1;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) err1;	ld	r6,8(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) err1;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) err1;	std	r6,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 9:	clrldi	r5,r5,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	/* Up to 15B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) .Lshort_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	mtocrf	0x01,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	bf	cr7*4+0,12f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) err1;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) err1;	lwz	r6,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	addi	r4,r4,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) err1;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) err1;	stw	r6,4(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 12:	bf	cr7*4+1,13f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) err1;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) err1;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 13:	bf	cr7*4+2,14f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) err1;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) err1;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 14:	bf	cr7*4+3,15f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) err1;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) err1;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 15:	li	r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) .Lunwind_stack_nonvmx_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	addi	r1,r1,STACKFRAMESIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	b	.Lnonvmx_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) .Lvmx_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) #ifdef CONFIG_ALTIVEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	mflr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	std	r0,16(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	stdu	r1,-STACKFRAMESIZE(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	bl	enter_vmx_usercopy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	cmpwi	cr1,r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	ld	r0,STACKFRAMESIZE+16(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	ld	r3,STK_REG(R31)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	ld	r4,STK_REG(R30)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	ld	r5,STK_REG(R29)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	mtlr	r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	 * We prefetch both the source and destination using enhanced touch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	 * instructions. We use a stream ID of 0 for the load side and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	 * 1 for the store side.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	clrrdi	r6,r4,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	clrrdi	r9,r3,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	ori	r9,r9,1		/* stream=1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	srdi	r7,r5,7		/* length in cachelines, capped at 0x3FF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	cmpldi	r7,0x3FF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	ble	1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	li	r7,0x3FF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 1:	lis	r0,0x0E00	/* depth=7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	sldi	r7,r7,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	or	r7,r7,r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	ori	r10,r7,1	/* stream=1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	lis	r8,0x8000	/* GO=1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	clrldi	r8,r8,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	/* setup read stream 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	dcbt	0,r6,0b01000   /* addr from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	dcbt	0,r7,0b01010   /* length and depth from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	/* setup write stream 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	dcbtst	0,r9,0b01000   /* addr to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	dcbtst	0,r10,0b01010  /* length and depth to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	eieio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	dcbt	0,r8,0b01010	/* all streams GO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	beq	cr1,.Lunwind_stack_nonvmx_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	 * If source and destination are not relatively aligned we use a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	 * slower permute loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	xor	r6,r4,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	rldicl.	r6,r6,0,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	bne	.Lvmx_unaligned_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	/* Get the destination 16B aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	clrldi	r6,r6,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	bf	cr7*4+3,1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) err3;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) err3;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	addi	r3,r3,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 1:	bf	cr7*4+2,2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) err3;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) err3;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 2:	bf	cr7*4+1,3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) err3;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 3:	bf	cr7*4+0,4f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) err3;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	addi	r4,r4,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) err3;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 4:	sub	r5,r5,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	/* Get the desination 128B aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	srdi	r7,r6,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	mtocrf	0x01,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	clrldi	r6,r6,(64-7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	li	r9,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	li	r10,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	li	r11,48
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 	bf	cr7*4+3,5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) err3;	stvx	v1,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 5:	bf	cr7*4+2,6f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) err3;	lvx	v0,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	addi	r4,r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) err3;	stvx	v1,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) err3;	stvx	v0,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 6:	bf	cr7*4+1,7f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) err3;	lvx	v3,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) err3;	lvx	v2,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) err3;	lvx	v1,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) err3;	lvx	v0,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	addi	r4,r4,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) err3;	stvx	v3,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) err3;	stvx	v2,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) err3;	stvx	v1,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) err3;	stvx	v0,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	addi	r3,r3,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 7:	sub	r5,r5,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	srdi	r6,r5,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	std	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	std	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	std	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	li	r12,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	li	r14,80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	li	r15,96
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 	li	r16,112
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 	 * Now do cacheline sized loads and stores. By this stage the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 	 * cacheline stores are also cacheline aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	.align	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) err4;	lvx	v7,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) err4;	lvx	v6,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) err4;	lvx	v5,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) err4;	lvx	v4,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) err4;	lvx	v3,r4,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) err4;	lvx	v2,r4,r14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) err4;	lvx	v1,r4,r15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) err4;	lvx	v0,r4,r16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 	addi	r4,r4,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) err4;	stvx	v7,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) err4;	stvx	v6,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) err4;	stvx	v5,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) err4;	stvx	v4,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) err4;	stvx	v3,r3,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) err4;	stvx	v2,r3,r14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) err4;	stvx	v1,r3,r15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) err4;	stvx	v0,r3,r16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	addi	r3,r3,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 	bdnz	8b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	ld	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	ld	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 	ld	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 	/* Up to 127B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	clrldi	r5,r5,(64-7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 	srdi	r6,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	bf	cr7*4+1,9f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) err3;	lvx	v3,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) err3;	lvx	v2,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) err3;	lvx	v1,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) err3;	lvx	v0,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	addi	r4,r4,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) err3;	stvx	v3,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) err3;	stvx	v2,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) err3;	stvx	v1,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) err3;	stvx	v0,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	addi	r3,r3,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 9:	bf	cr7*4+2,10f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) err3;	lvx	v0,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) 	addi	r4,r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) err3;	stvx	v1,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) err3;	stvx	v0,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 10:	bf	cr7*4+3,11f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) err3;	stvx	v1,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 	/* Up to 15B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 11:	clrldi	r5,r5,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	mtocrf	0x01,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	bf	cr7*4+0,12f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) err3;	ld	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	addi	r4,r4,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) err3;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 12:	bf	cr7*4+1,13f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) err3;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 13:	bf	cr7*4+2,14f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) err3;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) err3;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 14:	bf	cr7*4+3,15f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) err3;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) err3;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 15:	addi	r1,r1,STACKFRAMESIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 	b	exit_vmx_usercopy	/* tail call optimise */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) .Lvmx_unaligned_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 	/* Get the destination 16B aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	clrldi	r6,r6,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	bf	cr7*4+3,1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) err3;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 	addi	r4,r4,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) err3;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 	addi	r3,r3,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 1:	bf	cr7*4+2,2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) err3;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) err3;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) 2:	bf	cr7*4+1,3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) err3;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 3:	bf	cr7*4+0,4f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) err3;	lwz	r7,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) 	addi	r4,r4,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) err3;	stw	r7,4(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) 4:	sub	r5,r5,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) 	/* Get the desination 128B aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) 	srdi	r7,r6,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) 	mtocrf	0x01,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) 	clrldi	r6,r6,(64-7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) 	li	r9,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) 	li	r10,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) 	li	r11,48
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) 	LVS(v16,0,r4)		/* Setup permute control vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) err3;	lvx	v0,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) 	bf	cr7*4+3,5f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) 	VPERM(v8,v0,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) 	vor	v0,v1,v1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) 5:	bf	cr7*4+2,6f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) 	VPERM(v8,v0,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) err3;	lvx	v0,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) 	VPERM(v9,v1,v0,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) 	addi	r4,r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) err3;	stvx	v9,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) 6:	bf	cr7*4+1,7f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) err3;	lvx	v3,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) 	VPERM(v8,v0,v3,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) err3;	lvx	v2,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) 	VPERM(v9,v3,v2,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) err3;	lvx	v1,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) 	VPERM(v10,v2,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) err3;	lvx	v0,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) 	VPERM(v11,v1,v0,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) 	addi	r4,r4,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) err3;	stvx	v9,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) err3;	stvx	v10,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) err3;	stvx	v11,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) 	addi	r3,r3,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) 7:	sub	r5,r5,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) 	srdi	r6,r5,7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) 	std	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) 	std	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) 	std	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) 	li	r12,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) 	li	r14,80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) 	li	r15,96
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) 	li	r16,112
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) 	 * Now do cacheline sized loads and stores. By this stage the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) 	 * cacheline stores are also cacheline aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) 	.align	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) err4;	lvx	v7,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) 	VPERM(v8,v0,v7,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) err4;	lvx	v6,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) 	VPERM(v9,v7,v6,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) err4;	lvx	v5,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) 	VPERM(v10,v6,v5,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) err4;	lvx	v4,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) 	VPERM(v11,v5,v4,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) err4;	lvx	v3,r4,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) 	VPERM(v12,v4,v3,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) err4;	lvx	v2,r4,r14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) 	VPERM(v13,v3,v2,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) err4;	lvx	v1,r4,r15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) 	VPERM(v14,v2,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) err4;	lvx	v0,r4,r16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) 	VPERM(v15,v1,v0,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) 	addi	r4,r4,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) err4;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) err4;	stvx	v9,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) err4;	stvx	v10,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) err4;	stvx	v11,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) err4;	stvx	v12,r3,r12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) err4;	stvx	v13,r3,r14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) err4;	stvx	v14,r3,r15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) err4;	stvx	v15,r3,r16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) 	addi	r3,r3,128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) 	bdnz	8b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) 	ld	r14,STK_REG(R14)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) 	ld	r15,STK_REG(R15)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) 	ld	r16,STK_REG(R16)(r1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) 	/* Up to 127B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) 	clrldi	r5,r5,(64-7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) 	srdi	r6,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) 	bf	cr7*4+1,9f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) err3;	lvx	v3,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) 	VPERM(v8,v0,v3,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) err3;	lvx	v2,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) 	VPERM(v9,v3,v2,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) err3;	lvx	v1,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) 	VPERM(v10,v2,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) err3;	lvx	v0,r4,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) 	VPERM(v11,v1,v0,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) 	addi	r4,r4,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) err3;	stvx	v9,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) err3;	stvx	v10,r3,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) err3;	stvx	v11,r3,r11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) 	addi	r3,r3,64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) 9:	bf	cr7*4+2,10f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) 	VPERM(v8,v0,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) err3;	lvx	v0,r4,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) 	VPERM(v9,v1,v0,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) 	addi	r4,r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) err3;	stvx	v9,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) 10:	bf	cr7*4+3,11f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) err3;	lvx	v1,0,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) 	VPERM(v8,v0,v1,v16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) 	addi	r4,r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) err3;	stvx	v8,0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) 	/* Up to 15B to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) 11:	clrldi	r5,r5,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) 	addi	r4,r4,-16	/* Unwind the +16 load offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) 	mtocrf	0x01,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) 	bf	cr7*4+0,12f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) err3;	lwz	r6,4(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) 	addi	r4,r4,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) err3;	stw	r6,4(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) 12:	bf	cr7*4+1,13f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) err3;	lwz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) 	addi	r4,r4,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) err3;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) 13:	bf	cr7*4+2,14f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) err3;	lhz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) 	addi	r4,r4,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) err3;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) 14:	bf	cr7*4+3,15f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) err3;	lbz	r0,0(r4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) err3;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) 15:	addi	r1,r1,STACKFRAMESIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) 	b	exit_vmx_usercopy	/* tail call optimise */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) #endif /* CONFIG_ALTIVEC */