Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * arch/alpha/lib/ev6-stxcpy.S
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copy a null-terminated string from SRC to DST.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * This is an internal routine used by strcpy, stpcpy, and strcat.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * As such, it uses special linkage conventions to make implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * of these public functions more efficient.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * On input:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  *	t9 = return address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *	a0 = DST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  *	a1 = SRC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * On output:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  *	t12 = bitmask (with one bit set) indicating the last byte written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  *	a0  = unaligned address of the last *word* written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  * Furthermore, v0, a3-a5, t11, and t12 are untouched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * Much of the information about 21264 scheduling/coding comes from:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  *	Compiler Writer's Guide for the Alpha 21264
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  *	abbreviated as 'CWG' in other comments here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * Scheduling notation:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  *	E	- either cluster
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  * Try not to change the actual algorithm if possible for consistency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #include <asm/regdef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	.set noat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	.set noreorder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) /* There is a problem with either gdb (as of 4.16) or gas (as of 2.7) that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)    doesn't like putting the entry point for a procedure somewhere in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)    middle of the procedure descriptor.  Work around this by putting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)    aligned copy in its own procedure descriptor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	.ent stxcpy_aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	.align 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) stxcpy_aligned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	.frame sp, 0, t9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	.prologue 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	/* On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	   t0 == the first destination word for masking back in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	   t1 == the first source word.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	/* Create the 1st output word and detect 0's in the 1st input word.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	lda	t2, -1		# E : build a mask against false zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	mskqh	t2, a1, t2	# U :   detection in the src word (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	mskqh	t1, a1, t3	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	ornot	t1, t2, t2	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	mskql	t0, a1, t0	# U : assemble the first output word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	cmpbge	zero, t2, t8	# E : bits set iff null found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	or	t0, t3, t1	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	bne	t8, $a_eos	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	/* On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	   t0 == the first destination word for masking back in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	   t1 == a source word not containing a null.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	/* Nops here to separate store quads from load quads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) $a_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	stq_u	t1, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	addq	a0, 8, a0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	ldq_u	t1, 0(a1)	# L : Latency=3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	addq	a1, 8, a1	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	cmpbge	zero, t1, t8	# E : (3 cycle stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	beq	t8, $a_loop	# U : (stall for t8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	/* Take care of the final (partial) word store.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	   On entry to this basic block we have:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	   t1 == the source word containing the null
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	   t8 == the cmpbge mask that found it.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) $a_eos:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	negq	t8, t6		# E : find low bit set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	and	t8, t6, t12	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	/* For the sake of the cache, don't read a destination word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	   if we're not going to need it.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	and	t12, 0x80, t6	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	bne	t6, 1f		# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	/* We're doing a partial word store and so need to combine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	   our source and original destination words.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	ldq_u	t0, 0(a0)	# L : Latency=3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	subq	t12, 1, t6	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	zapnot	t1, t6, t1	# U : clear src bytes >= null (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	or	t12, t6, t8	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	zap	t0, t8, t0	# E : clear dst bytes <= null
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	or	t0, t1, t1	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 1:	stq_u	t1, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	ret	(t9)		# L0 : Latency=3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	.end stxcpy_aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	.align 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	.ent __stxcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	.globl __stxcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __stxcpy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	.frame sp, 0, t9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	.prologue 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/* Are source and destination co-aligned?  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	xor	a0, a1, t0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	unop			# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	and	t0, 7, t0	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	bne	t0, $unaligned	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	/* We are co-aligned; take care of a partial first word.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	ldq_u	t1, 0(a1)		# L : load first src word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	and	a0, 7, t0		# E : take care not to load a word ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	addq	a1, 8, a1		# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	beq	t0, stxcpy_aligned	# U : ... if we wont need it (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	ldq_u	t0, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	br	stxcpy_aligned	# L0 : Latency=3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* The source and destination are not co-aligned.  Align the destination
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)    and cope.  We have to be very careful about not reading too much and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)    causing a SEGV.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	.align 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) $u_head:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	/* We know just enough now to be able to assemble the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	   full source word.  We can still find a zero at the end of it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	   that prevents us from outputting the whole thing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	   On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	   t0 == the first dest word, for masking back in, if needed else 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	   t1 == the low bits of the first source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	   t6 == bytemask that is -1 in dest word bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	ldq_u	t2, 8(a1)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	addq	a1, 8, a1	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	extql	t1, a1, t1	# U : (stall on a1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	extqh	t2, a1, t4	# U : (stall on a1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	mskql	t0, a0, t0	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	or	t1, t4, t1	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	mskqh	t1, a0, t1	# U : (stall on t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	or	t0, t1, t1	# E : (stall on t1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	or	t1, t6, t6	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	cmpbge	zero, t6, t8	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	lda	t6, -1		# E : for masking just below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	bne	t8, $u_final	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	mskql	t6, a1, t6		# U : mask out the bits we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	or	t6, t2, t2		# E :   already extracted before (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	cmpbge	zero, t2, t8		# E :   testing eos (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	bne	t8, $u_late_head_exit	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	/* Finally, we've got all the stupid leading edge cases taken care
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	   of and we can set up to enter the main loop.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	stq_u	t1, 0(a0)	# L : store first output word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	addq	a0, 8, a0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	extql	t2, a1, t0	# U : position ho-bits of lo word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	ldq_u	t2, 8(a1)	# U : read next high-order source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	addq	a1, 8, a1	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	cmpbge	zero, t2, t8	# E : (stall for t2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	nop			# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	bne	t8, $u_eos	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	/* Unaligned copy main loop.  In order to avoid reading too much,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	   the loop is structured to detect zeros in aligned source words.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	   This has, unfortunately, effectively pulled half of a loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	   iteration out into the head and half into the tail, but it does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	   prevent nastiness from accumulating in the very thing we want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	   to run as fast as possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	   On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	   t0 == the shifted high-order bits from the previous source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	   t2 == the unshifted current source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	   We further know that t2 does not contain a null terminator.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	.align 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) $u_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	extqh	t2, a1, t1	# U : extract high bits for current word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	addq	a1, 8, a1	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	extql	t2, a1, t3	# U : extract low bits for next time (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	addq	a0, 8, a0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	or	t0, t1, t1	# E : current dst word now complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	ldq_u	t2, 0(a1)	# L : Latency=3 load high word for next time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	stq_u	t1, -8(a0)	# L : save the current word (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	mov	t3, t0		# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	cmpbge	zero, t2, t8	# E : test new word for eos
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	beq	t8, $u_loop	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	/* We've found a zero somewhere in the source word we just read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	   If it resides in the lower half, we have one (probably partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	   word to write out, and if it resides in the upper half, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	   have one full and one partial word left to write out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	   On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	   t0 == the shifted high-order bits from the previous source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	   t2 == the unshifted current source word.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) $u_eos:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	extqh	t2, a1, t1	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	or	t0, t1, t1	# E : first (partial) source word complete (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	cmpbge	zero, t1, t8	# E : is the null in this first bit? (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	bne	t8, $u_final	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) $u_late_head_exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	stq_u	t1, 0(a0)	# L : the null was in the high-order bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	addq	a0, 8, a0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	extql	t2, a1, t1	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	cmpbge	zero, t1, t8	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	/* Take care of a final (probably partial) result word.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	   On entry to this basic block:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	   t1 == assembled source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	   t8 == cmpbge mask that found the null.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) $u_final:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	negq	t8, t6		# E : isolate low bit set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	and	t6, t8, t12	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	and	t12, 0x80, t6	# E : avoid dest word load if we can (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	bne	t6, 1f		# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	ldq_u	t0, 0(a0)	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	subq	t12, 1, t6	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	or	t6, t12, t8	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	zapnot	t1, t6, t1	# U : kill source bytes >= null (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	zap	t0, t8, t0	# U : kill dest bytes <= null (2 cycle data stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	or	t0, t1, t1	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 1:	stq_u	t1, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	ret	(t9)		# L0 : Latency=3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	/* Unaligned copy entry point.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	.align 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) $unaligned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	ldq_u	t1, 0(a1)	# L : load first source word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	and	a0, 7, t4	# E : find dest misalignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	and	a1, 7, t5	# E : find src misalignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	/* Conditionally load the first destination word and a bytemask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	   with 0xff indicating that the destination byte is sacrosanct.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	mov	zero, t0	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	mov	zero, t6	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	beq	t4, 1f		# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	ldq_u	t0, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	lda	t6, -1		# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	mskql	t6, a0, t6	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	subq	a1, t4, a1	# E : sub dest misalignment from src addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	/* If source misalignment is larger than dest misalignment, we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	   extra startup checks to avoid SEGV.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	cmplt	t4, t5, t12	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	beq	t12, $u_head	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	lda	t2, -1		# E : mask out leading garbage in source
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	mskqh	t2, t5, t2	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	ornot	t1, t2, t3	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	cmpbge	zero, t3, t8	# E : is there a zero? (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	beq	t8, $u_head	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	/* At this point we've found a zero in the first partial word of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	   the source.  We need to isolate the valid source data and mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	   it into the original destination data.  (Incidentally, we know
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	   that we'll need at least one byte of that original dest word.) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	ldq_u	t0, 0(a0)	# L :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	negq	t8, t6		# E : build bitmask of bytes <= zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	and	t6, t8, t12	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	and	a1, 7, t5	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	subq	t12, 1, t6	# E :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	or	t6, t12, t8	# E : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	srl	t12, t5, t12	# U : adjust final null return value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	zapnot	t2, t8, t2	# U : prepare source word; mirror changes (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	and	t1, t2, t1	# E : to source validity mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	extql	t2, a1, t2	# U :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	extql	t1, a1, t1	# U : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	andnot	t0, t2, t0	# .. e1 : zero place for source to reside (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	or	t0, t1, t1	# e1    : and put it there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	stq_u	t1, 0(a0)	# .. e0 : (stall)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	ret	(t9)		# e1    :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	.end __stxcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)