Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Copyright (C) IBM Corporation, 2012
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Author: Anton Blanchard <anton@au.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/ppc_asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 	.section	".toc","aw"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) PPC64_CACHES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	.tc		ppc64_caches[TC],ppc64_caches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	.section	".text"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * __arch_clear_user: - Zero a block of memory in user space, with less checking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  * @to:   Destination address, in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * @n:    Number of bytes to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  * Zero a block of memory in user space.  Caller must check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  * the specified block with access_ok() before calling this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * Returns number of bytes that could not be cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * On success, this will be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	.macro err1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 100:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	EX_TABLE(100b,.Ldo_err1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	.macro err2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 200:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	EX_TABLE(200b,.Ldo_err2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	.macro err3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 300:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	EX_TABLE(300b,.Ldo_err3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) .Ldo_err1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	mr	r3,r8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) .Ldo_err2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	mtctr	r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) err3;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	addi	r3,r3,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	addi	r4,r4,-1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	bdnz	1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) .Ldo_err3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	mr	r3,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) _GLOBAL_TOC(__arch_clear_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	cmpdi	r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	li	r0,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	blt	.Lshort_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	mr	r8,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	mtocrf	0x01,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	clrldi	r6,r6,(64-3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	/* Get the destination 8 byte aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	bf	cr7*4+3,1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) err1;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	addi	r3,r3,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 1:	bf	cr7*4+2,2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) err1;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 2:	bf	cr7*4+1,3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) err1;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 3:	sub	r4,r4,r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	cmpdi	r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	cmpdi	cr1,r4,512
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	blt	.Lshort_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	bgt	cr1,.Llong_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) .Lmedium_clear:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	srdi	r6,r4,5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	/* Do 32 byte chunks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) err2;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) err2;	std	r0,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) err2;	std	r0,16(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) err2;	std	r0,24(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	addi	r3,r3,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	addi	r4,r4,-32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	bdnz	4b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .Lshort_clear:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	/* up to 31 bytes to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	cmpdi	r4,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	blt	6f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) err2;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) err2;	std	r0,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	addi	r4,r4,-16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	/* Up to 15 bytes to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 6:	mr	r8,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	clrldi	r4,r4,(64-4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	mtocrf	0x01,r4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	bf	cr7*4+0,7f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) err1;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 7:	bf	cr7*4+1,8f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) err1;	stw	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	addi	r3,r3,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 8:	bf	cr7*4+2,9f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) err1;	sth	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	addi	r3,r3,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 9:	bf	cr7*4+3,10f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) err1;	stb	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 10:	li	r3,0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	blr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) .Llong_clear:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	ld	r5,PPC64_CACHES@toc(r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	bf	cr7*4+0,11f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) err2;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	addi	r3,r3,8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	addi	r4,r4,-8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	/* Destination is 16 byte aligned, need to get it cache block aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 11:	lwz	r7,DCACHEL1LOGBLOCKSIZE(r5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	lwz	r9,DCACHEL1BLOCKSIZE(r5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	 * With worst case alignment the long clear loop takes a minimum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	 * of 1 byte less than 2 cachelines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	sldi	r10,r9,2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	cmpd	r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	blt	.Lmedium_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	neg	r6,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	addi	r10,r9,-1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	and.	r5,r6,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	beq	13f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	srdi	r6,r5,4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	mr	r8,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) err1;	std	r0,0(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) err1;	std	r0,8(r3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	addi	r3,r3,16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	bdnz	12b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	sub	r4,r4,r5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 13:	srd	r6,r4,r7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	mtctr	r6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	mr	r8,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) err1;	dcbz	0,r3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	add	r3,r3,r9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	bdnz	14b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	and	r4,r4,r10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	cmpdi	r4,32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	blt	.Lshort_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	b	.Lmedium_clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) EXPORT_SYMBOL(__arch_clear_user)