Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *    User Space Access Routines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *    Copyright (C) 2000-2002 Hewlett-Packard (John Marvin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *    Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *    Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *    Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *    Copyright (C) 2017 Helge Deller <deller@gmx.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *    Copyright (C) 2017 John David Anglin <dave.anglin@bell.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * These routines still have plenty of room for optimization
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * (word & doubleword load/store, dual issue, store hints, etc.).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * The following routines assume that space register 3 (sr3) contains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * the space id associated with the current users address space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <asm/assembly.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <asm/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	 * get_sr gets the appropriate space value into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	 * sr1 for kernel/user space access, depending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	 * on the flag stored in the task structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	.macro  get_sr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	mfctl       %cr30,%r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	ldw         TI_SEGMENT(%r1),%r22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	mfsp        %sr3,%r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	or,<>       %r22,%r0,%r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	copy        %r0,%r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	mtsp        %r1,%sr1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	.endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	 * unsigned long lclear_user(void *to, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	 * Returns 0 for success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	 * otherwise, returns number of bytes not transferred.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) ENTRY_CFI(lclear_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	comib,=,n   0,%r25,$lclu_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	get_sr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) $lclu_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	addib,<>    -1,%r25,$lclu_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 1:      stbs,ma     %r0,1(%sr1,%r26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) $lclu_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	bv          %r0(%r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	copy        %r25,%r28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 2:	b           $lclu_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	ldo         1(%r25),%r25
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	ASM_EXCEPTIONTABLE_ENTRY(1b,2b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) ENDPROC_CFI(lclear_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	 * long lstrnlen_user(char *s, long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	 * Returns 0 if exception before zero byte or reaching N,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	 *         N+1 if N would be exceeded,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	 *         else strlen + 1 (i.e. includes zero byte).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) ENTRY_CFI(lstrnlen_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	comib,=     0,%r25,$lslen_nzero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	copy	    %r26,%r24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	get_sr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 1:      ldbs,ma     1(%sr1,%r26),%r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) $lslen_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	comib,=,n   0,%r1,$lslen_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	addib,<>    -1,%r25,$lslen_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 2:      ldbs,ma     1(%sr1,%r26),%r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) $lslen_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	bv          %r0(%r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	sub	    %r26,%r24,%r28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) $lslen_nzero:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	b           $lslen_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	ldo         1(%r26),%r26 /* special case for N == 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 3:      b	    $lslen_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	copy        %r24,%r26    /* reset r26 so 0 is returned on fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	ASM_EXCEPTIONTABLE_ENTRY(1b,3b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	ASM_EXCEPTIONTABLE_ENTRY(2b,3b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ENDPROC_CFI(lstrnlen_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)  * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)  * Inputs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  * - sr1 already contains space of source region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  * - sr2 already contains space of destination region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * - number of bytes that could not be copied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  *   On success, this will be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * This code is based on a C-implementation of a copy routine written by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * Randolph Chung, which in turn was derived from the glibc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * Several strategies are tried to try to get the best performance for various
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * conditions. In the optimal case, we copy by loops that copy 32- or 16-bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  * at a time using general registers.  Unaligned copies are handled either by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  * aligning the destination and then using shift-and-write method, or in a few
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  * cases by falling back to a byte-at-a-time copy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  * Testing with various alignments and buffer sizes shows that this code is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  * often >10x faster than a simple byte-at-a-time copy, even for strangely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)  * aligned operands. It is interesting to note that the glibc version of memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)  * (written in C) is actually quite fast already. This routine is able to beat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)  * it by 30-40% for aligned copies because of the loop unrolling, but in some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)  * cases the glibc version is still slightly faster. This lends more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)  * credibility that gcc can generate very good code as long as we are careful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)  * Possible optimizations:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)  * - add cache prefetching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)  * - try not to use the post-increment address modifiers; they may create
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)  *   additional interlocks. Assumption is that those were only efficient on old
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)  *   machines (pre PA8000 processors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	dst = arg0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	src = arg1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	len = arg2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	end = arg3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	t1  = r19
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	t2  = r20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	t3  = r21
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	t4  = r22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	srcspc = sr1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	dstspc = sr2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	t0 = r1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	a1 = t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	a2 = t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	a3 = t3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	a0 = t4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	save_src = ret0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	save_dst = ret1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	save_len = r31
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) ENTRY_CFI(pa_memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	/* Last destination address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	add	dst,len,end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	/* short copy with less than 16 bytes? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	cmpib,COND(>>=),n 15,len,.Lbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	/* same alignment? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	xor	src,dst,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	extru	t0,31,2,t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	cmpib,<>,n  0,t1,.Lunaligned_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	/* only do 64-bit copies if we can get aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	extru	t0,31,3,t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	cmpib,<>,n  0,t1,.Lalign_loop32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	/* loop until we are 64-bit aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) .Lalign_loop64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	extru	dst,31,3,t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	cmpib,=,n	0,t1,.Lcopy_loop_16_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 20:	ldb,ma	1(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 21:	stb,ma	t1,1(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	b	.Lalign_loop64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	ldo	-1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .Lcopy_loop_16_start:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	ldi	31,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) .Lcopy_loop_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	cmpb,COND(>>=),n t0,len,.Lword_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 10:	ldd	0(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 11:	ldd	8(srcspc,src),t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	ldo	16(src),src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 12:	std,ma	t1,8(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 13:	std,ma	t2,8(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 14:	ldd	0(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 15:	ldd	8(srcspc,src),t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	ldo	16(src),src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 16:	std,ma	t1,8(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 17:	std,ma	t2,8(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy16_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy16_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	b	.Lcopy_loop_16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	ldo	-32(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) .Lword_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	cmpib,COND(>>=),n 3,len,.Lbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 20:	ldw,ma	4(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 21:	stw,ma	t1,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	b	.Lword_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	ldo	-4(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #endif /* CONFIG_64BIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	/* loop until we are 32-bit aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) .Lalign_loop32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	extru	dst,31,2,t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	cmpib,=,n	0,t1,.Lcopy_loop_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 20:	ldb,ma	1(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 21:	stb,ma	t1,1(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	b	.Lalign_loop32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	ldo	-1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) .Lcopy_loop_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	cmpib,COND(>>=),n 15,len,.Lbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 10:	ldw	0(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 11:	ldw	4(srcspc,src),t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 12:	stw,ma	t1,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 13:	stw,ma	t2,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 14:	ldw	8(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 15:	ldw	12(srcspc,src),t2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	ldo	16(src),src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 16:	stw,ma	t1,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 17:	stw,ma	t2,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy8_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy8_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	b	.Lcopy_loop_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	ldo	-16(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) .Lbyte_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	cmpclr,COND(<>) len,%r0,%r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	b,n	.Lcopy_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 20:	ldb	0(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	ldo	1(src),src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 21:	stb,ma	t1,1(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	b	.Lbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	ldo	-1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) .Lcopy_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	bv	%r0(%r2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	sub	end,dst,ret0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	/* src and dst are not aligned the same way. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	/* need to go the hard way */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) .Lunaligned_copy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	/* align until dst is 32bit-word-aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	extru	dst,31,2,t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	cmpib,=,n	0,t1,.Lcopy_dstaligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 20:	ldb	0(srcspc,src),t1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	ldo	1(src),src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 21:	stb,ma	t1,1(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	b	.Lunaligned_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	ldo	-1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) .Lcopy_dstaligned:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	/* store src, dst and len in safe place */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	copy	src,save_src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	copy	dst,save_dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	copy	len,save_len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	/* len now needs give number of words to copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	SHRREG	len,2,len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	 * Copy from a not-aligned src to an aligned dst using shifts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	 * Handles 4 words per loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	depw,z src,28,2,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	subi 32,t0,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	mtsar t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	extru len,31,2,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	cmpib,= 2,t0,.Lcase2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	/* Make src aligned by rounding it down.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	depi 0,31,2,src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	cmpiclr,<> 3,t0,%r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	b,n .Lcase3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	cmpiclr,<> 1,t0,%r0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	b,n .Lcase1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) .Lcase0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	cmpb,COND(=) %r0,len,.Lcda_finish
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 1:	ldw,ma 4(srcspc,src), a3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 1:	ldw,ma 4(srcspc,src), a0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	b,n .Ldo3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) .Lcase1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 1:	ldw,ma 4(srcspc,src), a2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 1:	ldw,ma 4(srcspc,src), a3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	ldo -1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	cmpb,COND(=),n %r0,len,.Ldo0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) .Ldo4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 1:	ldw,ma 4(srcspc,src), a0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	shrpw a2, a3, %sar, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 1:	stw,ma t0, 4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) .Ldo3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 1:	ldw,ma 4(srcspc,src), a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	shrpw a3, a0, %sar, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 1:	stw,ma t0, 4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) .Ldo2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 1:	ldw,ma 4(srcspc,src), a2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	shrpw a0, a1, %sar, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 1:	stw,ma t0, 4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) .Ldo1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 1:	ldw,ma 4(srcspc,src), a3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	shrpw a1, a2, %sar, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 1:	stw,ma t0, 4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	ldo -4(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	cmpb,COND(<>) %r0,len,.Ldo4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) .Ldo0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	shrpw a2, a3, %sar, t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 1:	stw,ma t0, 4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) .Lcda_rdfault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) .Lcda_finish:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	/* calculate new src, dst and len and jump to byte-copy loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	sub	dst,save_dst,t0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	add	save_src,t0,src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	b	.Lbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	sub	save_len,t0,len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) .Lcase3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 1:	ldw,ma 4(srcspc,src), a0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 1:	ldw,ma 4(srcspc,src), a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	b .Ldo2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	ldo 1(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) .Lcase2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 1:	ldw,ma 4(srcspc,src), a1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 1:	ldw,ma 4(srcspc,src), a2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	b .Ldo1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	ldo 2(len),len
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	/* fault exception fixup handlers: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) .Lcopy16_fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	b	.Lcopy_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 10:	std,ma	t1,8(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) .Lcopy8_fault:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	b	.Lcopy_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 10:	stw,ma	t1,4(dstspc,dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ENDPROC_CFI(pa_memcpy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 	.end