^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * M7memcpy: Optimized SPARC M7 memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) .file "M7memcpy.S"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * memcpy(s1, s2, len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Copy s2 to s1, always copy n bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Note: this C code does not work for overlapped copies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Fast assembler language version of the following C-program for memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * which represents the `standard' for the C-library.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * void *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * memcpy(void *s, const void *s0, size_t n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * if (n != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * char *s1 = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * const char *s2 = s0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * *s1++ = *s2++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * } while (--n != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * return (s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * SPARC T7/M7 Flow :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * if (count < SMALL_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * if count < SHORTCOPY (SHORTCOPY=3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * copy bytes; exit with dst addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * if src & dst aligned on word boundary but not long word boundary,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * copy with ldw/stw; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * if src & dst aligned on long word boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * copy with ldx/stx; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * if src & dst not aligned and length <= SHORTCHECK (SHORTCHECK=14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * copy bytes; exit with dst addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * move enough bytes to get src to word boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * if dst now on word boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * move_words:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * copy words; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * if dst now on half word boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * load words, shift half words, store words; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * if dst on byte 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * load words, shift 3 bytes, store words; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * if dst on byte 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * load words, shift 1 byte, store words; branch to finish_up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * finish_up:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * copy bytes; exit with dst addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * } else { More than SMALL_MAX bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * move bytes until dst is on long word boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * if( src is on long word boundary ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * if (count < MED_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * finish_long: src/dst aligned on 8 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * copy with ldx/stx in 8-way unrolled loop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * copy final 0-63 bytes; exit with dst addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * } else { src/dst aligned; count > MED_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * align dst on 64 byte boundary; for main data movement:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * prefetch src data to L2 cache; let HW prefetch move data to L1 cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * Use BIS (block initializing store) to avoid copying store cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * lines from memory. But pre-store first element of each cache line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * ST_CHUNK lines in advance of the rest of that cache line. That
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * gives time for replacement cache lines to be written back without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * excess STQ and Miss Buffer filling. Repeat until near the end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * then finish up storing before going to finish_long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * } else { src/dst not aligned on 8 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * if src is word aligned and count < MED_WMAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * move words in 8-way unrolled loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * move final 0-31 bytes; exit with dst addr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * if count < MED_UMAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * use alignaddr/faligndata combined with ldd/std in 8-way
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * unrolled loop to move data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * go to unalign_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * setup alignaddr for faligndata instructions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * align dst on 64 byte boundary; prefetch src data to L1 cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * loadx8, falign, block-store, prefetch loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * (only use block-init-store when src/dst on 8 byte boundaries.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * unalign_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * move remaining bytes for unaligned cases. exit with dst addr.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #include <asm/visasm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #include <asm/asi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #if !defined(EX_LD) && !defined(EX_ST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define NON_USER_COPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #ifndef EX_LD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define EX_LD(x,y) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #ifndef EX_LD_FP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define EX_LD_FP(x,y) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #ifndef EX_ST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define EX_ST(x,y) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #ifndef EX_ST_FP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define EX_ST_FP(x,y) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #ifndef EX_RETVAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define EX_RETVAL(x) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #ifndef LOAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define LOAD(type,addr,dest) type [addr], dest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #ifndef STORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define STORE(type,src,addr) type src, [addr]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * line as "least recently used" which means if many threads are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * active, it has a high probability of being pushed out of the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * between the first initializing store and the final stores.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * marks the cache line as "most recently used" for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * but the last cache line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #ifndef STORE_ASI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define STORE_ASI 0x80 /* ASI_P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #ifndef STORE_MRU_ASI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define STORE_MRU_ASI 0x80 /* ASI_P */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #ifndef STORE_INIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #ifndef STORE_INIT_MRU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define STORE_INIT_MRU(src,addr) stxa src, [addr] STORE_MRU_ASI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #ifndef FUNC_NAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #define FUNC_NAME M7memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) #ifndef PREAMBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define PREAMBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define BLOCK_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #define SHORTCOPY 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define SHORTCHECK 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #define SHORT_LONG 64 /* max copy for short longword-aligned case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /* must be at least 64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #define SMALL_MAX 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #define MED_UMAX 1024 /* max copy for medium un-aligned case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #define MED_WMAX 1024 /* max copy for medium word-aligned case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #define MED_MAX 1024 /* max copy for medium longword-aligned case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #define ST_CHUNK 24 /* ST_CHUNK - block of values for BIS Store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #define ALIGN_PRE 24 /* distance for aligned prefetch loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) .register %g2,#scratch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) .section ".text"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) .global FUNC_NAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) .type FUNC_NAME, #function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) FUNC_NAME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) srlx %o2, 31, %g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) cmp %g2, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) tne %xcc, 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) PREAMBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) mov %o0, %g1 ! save %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) brz,pn %o2, .Lsmallx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) cmp %o2, 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) ble,pn %icc, .Ltiny_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) cmp %o2, 19
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) ble,pn %icc, .Lsmall_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) or %o0, %o1, %g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) cmp %o2, SMALL_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) bl,pn %icc, .Lmedium_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) .Lmedium:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) neg %o0, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) andcc %o5, 7, %o5 ! bytes till DST 8 byte aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) brz,pt %o5, .Ldst_aligned_on_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) ! %o5 has the bytes to be written in partial store.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) sub %o2, %o5, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) sub %o1, %o0, %o1 ! %o1 gets the difference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 7: ! dst aligning loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) add %o1, %o0, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5) ! load one byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) subcc %o5, 1, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) bgu,pt %xcc, 7b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) add %o0, 1, %o0 ! advance dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) add %o1, %o0, %o1 ! restore %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) .Ldst_aligned_on_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) andcc %o1, 7, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) brnz,pt %o5, .Lsrc_dst_unaligned_on_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) .Lsrc_dst_aligned_on_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) ! check if we are copying MED_MAX or more bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) set MED_MAX, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) cmp %o2, %o3 ! limit to store buffer size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) bgu,pn %xcc, .Llarge_align8_copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * Special case for handling when src and dest are both long word aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * and total data to move is less than MED_MAX bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) .Lmedlong:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) subcc %o2, 63, %o2 ! adjust length to allow cc test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) ble,pn %xcc, .Lmedl63 ! skip big loop if less than 64 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) .Lmedl64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63) ! load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) subcc %o2, 64, %o2 ! decrement length count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64) ! and store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56) ! a block of 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_63_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) EX_LD(LOAD(ldx, %o1+24, %o3), memcpy_retl_o2_plus_63_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) EX_ST(STORE(stx, %o3, %o0+24), memcpy_retl_o2_plus_63_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_63_32)! load and store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) EX_ST(STORE(stx, %o4, %o0+32), memcpy_retl_o2_plus_63_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) EX_LD(LOAD(ldx, %o1+40, %o3), memcpy_retl_o2_plus_63_24)! a block of 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) add %o1, 64, %o1 ! increase src ptr by 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) EX_ST(STORE(stx, %o3, %o0+40), memcpy_retl_o2_plus_63_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_63_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) add %o0, 64, %o0 ! increase dst ptr by 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_63_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_63_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) bgu,pt %xcc, .Lmedl64 ! repeat if at least 64 bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_63_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) .Lmedl63:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) addcc %o2, 32, %o2 ! adjust remaining count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) ble,pt %xcc, .Lmedl31 ! to skip if 31 or fewer bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_31) ! load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) sub %o2, 32, %o2 ! decrement length count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_31_32) ! and store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_31_24) ! a block of 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) add %o1, 32, %o1 ! increase src ptr by 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_31_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_31_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) add %o0, 32, %o0 ! increase dst ptr by 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_31_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_31_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_31_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) .Lmedl31:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) addcc %o2, 16, %o2 ! adjust remaining count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) ble,pt %xcc, .Lmedl15 ! skip if 15 or fewer bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) nop !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) add %o1, 16, %o1 ! increase src ptr by 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) sub %o2, 16, %o2 ! decrease count by 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_15_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) add %o0, 16, %o0 ! increase dst ptr by 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_15_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) .Lmedl15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) addcc %o2, 15, %o2 ! restore count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) bz,pt %xcc, .Lsmallx ! exit if finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) cmp %o2, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) blt,pt %xcc, .Lmedw7 ! skip if 7 or fewer bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) tst %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) ! load 8 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) add %o1, 8, %o1 ! increase src ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) add %o0, 8, %o0 ! increase dst ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) subcc %o2, 8, %o2 ! decrease count by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) bnz,pn %xcc, .Lmedw7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) ! and store 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) retl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) mov EX_RETVAL(%g1), %o0 ! restore %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) .Lsrc_dst_unaligned_on_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) ! DST is 8-byte aligned, src is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) andcc %o1, 0x3, %o5 ! test word alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) bnz,pt %xcc, .Lunalignsetup ! branch to skip if not word aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * Handle all cases where src and dest are aligned on word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * boundaries. Use unrolled loops for better performance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * This option wins over standard large data move when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * source and destination is in cache for.Lmedium
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * to short data moves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) set MED_WMAX, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) cmp %o2, %o3 ! limit to store buffer size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) bge,pt %xcc, .Lunalignrejoin ! otherwise rejoin main loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) subcc %o2, 31, %o2 ! adjust length to allow cc test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) ! for end of loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) ble,pt %xcc, .Lmedw31 ! skip big loop if less than 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) .Lmedw32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_31)! move a block of 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) subcc %o2, 32, %o2 ! decrement length count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) EX_LD(LOAD(ld, %o1+8, %o4), memcpy_retl_o2_plus_31_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) EX_LD(LOAD(ld, %o1+12, %o4), memcpy_retl_o2_plus_31_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) EX_ST(STORE(stx, %o5, %o0+8), memcpy_retl_o2_plus_31_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) add %o1, 32, %o1 ! increase src ptr by 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) EX_LD(LOAD(ld, %o1-16, %o4), memcpy_retl_o2_plus_31_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) EX_LD(LOAD(ld, %o1-12, %o4), memcpy_retl_o2_plus_31_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) EX_ST(STORE(stx, %o5, %o0+16), memcpy_retl_o2_plus_31_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) add %o0, 32, %o0 ! increase dst ptr by 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_31_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_31_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) bgu,pt %xcc, .Lmedw32 ! repeat if at least 32 bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_31_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) .Lmedw31:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) addcc %o2, 31, %o2 ! restore count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) bz,pt %xcc, .Lsmallx ! exit if finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) cmp %o2, 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) blt,pt %xcc, .Lmedw15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)! move a block of 16 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) subcc %o2, 16, %o2 ! decrement length count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) add %o1, 16, %o1 ! increase src ptr by 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) add %o0, 16, %o0 ! increase dst ptr by 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) sllx %o4, 32, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) .Lmedw15:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) bz,pt %xcc, .Lsmallx ! exit if finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) cmp %o2, 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) blt,pn %xcc, .Lmedw7 ! skip if 7 or fewer bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) tst %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) subcc %o2, 8, %o2 ! decrease count by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) EX_ST(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_8)! and store 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) add %o1, 8, %o1 ! increase src ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) EX_LD(LOAD(ld, %o1-4, %o3), memcpy_retl_o2_plus_4) ! load 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) add %o0, 8, %o0 ! increase dst ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) EX_ST(STORE(stw, %o3, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) bz,pt %xcc, .Lsmallx ! exit if finished
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) .Lmedw7: ! count is ge 1, less than 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) cmp %o2, 4 ! check for 4 bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) blt,pn %xcc, .Lsmallleft3 ! skip if 3 or fewer bytes left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) nop !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) add %o1, 4, %o1 ! increase src ptr by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) add %o0, 4, %o0 ! increase dst ptr by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) subcc %o2, 4, %o2 ! decrease count by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) bnz .Lsmallleft3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) EX_ST(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) retl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) mov EX_RETVAL(%g1), %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) .Llarge_align8_copy: ! Src and dst share 8 byte alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) ! align dst to 64 byte boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) andcc %o0, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) brz,pn %o3, .Laligned_to_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) andcc %o0, 8, %o3 ! odd long words to move?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) brz,pt %o3, .Laligned_to_16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) sub %o2, 8, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) add %o1, 8, %o1 ! increment src ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) add %o0, 8, %o0 ! increment dst ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) .Laligned_to_16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) andcc %o0, 16, %o3 ! pair of long words to move?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) brz,pt %o3, .Laligned_to_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) sub %o2, 16, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) add %o1, 16, %o1 ! increment src ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) add %o0, 16, %o0 ! increment dst ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) .Laligned_to_32:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) andcc %o0, 32, %o3 ! four long words to move?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) brz,pt %o3, .Laligned_to_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) sub %o2, 32, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) add %o1, 32, %o1 ! increment src ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) add %o0, 32, %o0 ! increment dst ptr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) .Laligned_to_64:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) ! Using block init store (BIS) instructions to avoid fetching cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ! lines from memory. Use ST_CHUNK stores to first element of each cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) ! line (similar to prefetching) to avoid overfilling STQ or miss buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) ! Gives existing cache lines time to be moved out of L1/L2/L3 cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) ! Initial stores using MRU version of BIS to keep cache line in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) ! cache until we are ready to store final element of cache line.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) ! Then store last element using the LRU version of BIS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) andn %o2, 0x3f, %o5 ! %o5 is multiple of block size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) and %o2, 0x3f, %o2 ! residue bytes in %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) ! We use STORE_MRU_ASI for the first seven stores to each cache line
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) ! followed by STORE_ASI (mark as LRU) for the last store. That
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) ! mixed approach reduces the probability that the cache line is removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) ! before we finish setting it, while minimizing the effects on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) ! other cached values during a large memcpy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) ! ST_CHUNK batches up initial BIS operations for several cache lines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) ! to allow multiple requests to not be blocked by overflowing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) ! the store miss buffer. Then the matching stores for all those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) ! BIS operations are executed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) !
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) sub %o0, 8, %o0 ! adjust %o0 for ASI alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) .Lalign_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) cmp %o5, ST_CHUNK*64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) blu,pt %xcc, .Lalign_loop_fin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) mov ST_CHUNK,%o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) .Lalign_loop_start:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) prefetch [%o1 + (ALIGN_PRE * BLOCK_SIZE)], 21
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) subcc %o3, 1, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) add %o1, 64, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) bgu %xcc,.Lalign_loop_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) add %o0, 56, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) mov ST_CHUNK,%o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) sllx %o3, 6, %o4 ! ST_CHUNK*64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) sub %o1, %o4, %o1 ! reset %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) sub %o0, %o4, %o0 ! reset %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) .Lalign_loop_rest:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) add %o0, 16, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) subcc %o3, 1, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) add %o1, 64, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) sub %o5, 64, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) bgu %xcc,.Lalign_loop_rest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) ! mark cache line as LRU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) EX_ST(STORE_INIT(%o4, %o0), memcpy_retl_o2_plus_o5_plus_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) cmp %o5, ST_CHUNK*64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) bgu,pt %xcc, .Lalign_loop_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) mov ST_CHUNK,%o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) cmp %o5, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) beq .Lalign_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) .Lalign_loop_fin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) EX_ST(STORE(stx, %o4, %o0+8+8), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) EX_ST(STORE(stx, %o4, %o0+8+16), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) subcc %o5, 64, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) EX_ST(STORE(stx, %o4, %o0+8+24), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) EX_ST(STORE(stx, %o4, %o0+8+32), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) EX_ST(STORE(stx, %o4, %o0+8+40), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) add %o1, 64, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) EX_ST(STORE(stx, %o4, %o0+8+48), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) add %o0, 64, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) bgu %xcc,.Lalign_loop_fin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_o5_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) .Lalign_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) add %o0, 8, %o0 ! restore %o0 from ASI alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) membar #StoreStore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) sub %o2, 63, %o2 ! adjust length to allow cc test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) ba .Lmedl63 ! in .Lmedl63
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) .align 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) .Lunalignsetup:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) .Lunalignrejoin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) mov %g1, %o3 ! save %g1 as VISEntryHalf clobbers it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) #ifdef NON_USER_COPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) VISEntryHalfFast(.Lmedium_vis_entry_fail_cp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) VISEntryHalf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) mov %o3, %g1 ! restore %g1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) set MED_UMAX, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) cmp %o2, %o3 ! check for.Lmedium unaligned limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) bge,pt %xcc,.Lunalign_large
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) prefetch [%o1 + (4 * BLOCK_SIZE)], 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) andn %o2, 0x3f, %o5 ! %o5 is multiple of block size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) and %o2, 0x3f, %o2 ! residue bytes in %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) cmp %o2, 8 ! Insure we do not load beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) bgt .Lunalign_adjust ! end of source buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) andn %o1, 0x7, %o4 ! %o4 has long word aligned src address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) add %o2, 64, %o2 ! adjust to leave loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) sub %o5, 64, %o5 ! early if necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) .Lunalign_adjust:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) alignaddr %o1, %g0, %g0 ! generate %gsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) add %o1, %o5, %o1 ! advance %o1 to after blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) .Lunalign_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) faligndata %f0, %f2, %f16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) EX_LD_FP(LOAD(ldd, %o4+16, %f4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) subcc %o5, BLOCK_SIZE, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) faligndata %f2, %f4, %f18
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) EX_LD_FP(LOAD(ldd, %o4+24, %f6), memcpy_retl_o2_plus_o5_plus_56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) EX_ST_FP(STORE(std, %f18, %o0+8), memcpy_retl_o2_plus_o5_plus_56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) faligndata %f4, %f6, %f20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) EX_LD_FP(LOAD(ldd, %o4+32, %f8), memcpy_retl_o2_plus_o5_plus_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) EX_ST_FP(STORE(std, %f20, %o0+16), memcpy_retl_o2_plus_o5_plus_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) faligndata %f6, %f8, %f22
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) EX_LD_FP(LOAD(ldd, %o4+40, %f10), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) EX_ST_FP(STORE(std, %f22, %o0+24), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) faligndata %f8, %f10, %f24
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) EX_LD_FP(LOAD(ldd, %o4+48, %f12), memcpy_retl_o2_plus_o5_plus_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) EX_ST_FP(STORE(std, %f24, %o0+32), memcpy_retl_o2_plus_o5_plus_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) faligndata %f10, %f12, %f26
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) EX_LD_FP(LOAD(ldd, %o4+56, %f14), memcpy_retl_o2_plus_o5_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) add %o4, BLOCK_SIZE, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) EX_ST_FP(STORE(std, %f26, %o0+40), memcpy_retl_o2_plus_o5_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) faligndata %f12, %f14, %f28
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) EX_ST_FP(STORE(std, %f28, %o0+48), memcpy_retl_o2_plus_o5_plus_16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) faligndata %f14, %f0, %f30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) EX_ST_FP(STORE(std, %f30, %o0+56), memcpy_retl_o2_plus_o5_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) add %o0, BLOCK_SIZE, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) bgu,pt %xcc, .Lunalign_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) prefetch [%o4 + (5 * BLOCK_SIZE)], 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ba .Lunalign_done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) .Lunalign_large:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) andcc %o0, 0x3f, %o3 ! is dst 64-byte block aligned?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) bz %xcc, .Lunalignsrc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) sub %o3, 64, %o3 ! %o3 will be multiple of 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) neg %o3 ! bytes until dest is 64 byte aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) sub %o2, %o3, %o2 ! update cnt with bytes to be moved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) ! Move bytes according to source alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) andcc %o1, 0x1, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) bnz %xcc, .Lunalignbyte ! check for byte alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) andcc %o1, 2, %o5 ! check for half word alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) bnz %xcc, .Lunalignhalf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) ! Src is word aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) .Lunalignword:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) EX_LD_FP(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 4 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) add %o1, 8, %o1 ! increase src ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) EX_ST_FP(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_o3) ! and store 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) subcc %o3, 8, %o3 ! decrease count by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) EX_LD_FP(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_o3_plus_4)! load 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) add %o0, 8, %o0 ! increase dst ptr by 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) bnz %xcc, .Lunalignword
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) EX_ST_FP(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_o3_plus_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) ba .Lunalignsrc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) ! Src is half-word aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) .Lunalignhalf:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) EX_LD_FP(LOAD(lduh, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 2 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) sllx %o4, 32, %o5 ! shift left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) EX_LD_FP(LOAD(lduw, %o1+2, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) sllx %o5, 16, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) EX_LD_FP(LOAD(lduh, %o1+6, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) add %o1, 8, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) subcc %o3, 8, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) bnz %xcc, .Lunalignhalf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ba .Lunalignsrc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) ! Src is Byte aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) .Lunalignbyte:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) sub %o0, %o1, %o0 ! share pointer advance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) .Lunalignbyte_loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) EX_LD_FP(LOAD(ldub, %o1, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) sllx %o4, 56, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) EX_LD_FP(LOAD(lduh, %o1+1, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) sllx %o4, 40, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) EX_LD_FP(LOAD(lduh, %o1+3, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) sllx %o4, 24, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) EX_LD_FP(LOAD(lduh, %o1+5, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) sllx %o4, 8, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) EX_LD_FP(LOAD(ldub, %o1+7, %o4), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) or %o4, %o5, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) add %o0, %o1, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) sub %o0, %o1, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) subcc %o3, 8, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) bnz %xcc, .Lunalignbyte_loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) add %o1, 8, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) add %o0,%o1, %o0 ! restore pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) ! Destination is now block (64 byte aligned)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) .Lunalignsrc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) andn %o2, 0x3f, %o5 ! %o5 is multiple of block size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) and %o2, 0x3f, %o2 ! residue bytes in %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) add %o2, 64, %o2 ! Insure we do not load beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) sub %o5, 64, %o5 ! end of source buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) andn %o1, 0x7, %o4 ! %o4 has long word aligned src address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) alignaddr %o1, %g0, %g0 ! generate %gsr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) add %o1, %o5, %o1 ! advance %o1 to after blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) EX_LD_FP(LOAD(ldd, %o4, %f14), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) add %o4, 8, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) .Lunalign_sloop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) EX_LD_FP(LOAD(ldd, %o4, %f16), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) faligndata %f14, %f16, %f0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) EX_LD_FP(LOAD(ldd, %o4+8, %f18), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) faligndata %f16, %f18, %f2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) EX_LD_FP(LOAD(ldd, %o4+16, %f20), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) faligndata %f18, %f20, %f4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) EX_ST_FP(STORE(std, %f0, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) subcc %o5, 64, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) EX_LD_FP(LOAD(ldd, %o4+24, %f22), memcpy_retl_o2_plus_o5_plus_56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) faligndata %f20, %f22, %f6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) EX_ST_FP(STORE(std, %f2, %o0+8), memcpy_retl_o2_plus_o5_plus_56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) EX_LD_FP(LOAD(ldd, %o4+32, %f24), memcpy_retl_o2_plus_o5_plus_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) faligndata %f22, %f24, %f8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) EX_ST_FP(STORE(std, %f4, %o0+16), memcpy_retl_o2_plus_o5_plus_48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) faligndata %f24, %f26, %f10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) faligndata %f26, %f28, %f12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) add %o4, 64, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) faligndata %f28, %f30, %f14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) add %o0, 64, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) fsrc2 %f30, %f14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) bgu,pt %xcc, .Lunalign_sloop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) prefetch [%o4 + (8 * BLOCK_SIZE)], 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) .Lunalign_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) ! Handle trailing bytes, 64 to 127
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) ! Dest long word aligned, Src not long word aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) cmp %o2, 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) bleu %xcc, .Lunalign_short
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) andn %o2, 0x7, %o5 ! %o5 is multiple of 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) and %o2, 0x7, %o2 ! residue bytes in %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) add %o2, 8, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) sub %o5, 8, %o5 ! insure we do not load past end of src
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) andn %o1, 0x7, %o4 ! %o4 has long word aligned src address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) add %o1, %o5, %o1 ! advance %o1 to after multiple of 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)! fetch partialword
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) .Lunalign_by8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) add %o4, 8, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) faligndata %f0, %f2, %f16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) subcc %o5, 8, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) fsrc2 %f2, %f0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) bgu,pt %xcc, .Lunalign_by8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) add %o0, 8, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) .Lunalign_short:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) #ifdef NON_USER_COPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) VISExitHalfFast
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) VISExitHalf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) ba .Lsmallrest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * This is a special case of nested memcpy. This can happen when kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * calls unaligned memcpy back to back without saving FP registers. We need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * traps(context switch) to save/restore FP registers. If the kernel calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * memcpy without this trap sequence we will hit FP corruption. Let's use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * the normal integer load/store method in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) #ifdef NON_USER_COPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) .Lmedium_vis_entry_fail_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) or %o0, %o1, %g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) .Lmedium_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) LOAD(prefetch, %o1 + 0x40, #n_reads_strong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) andcc %g2, 0x7, %g0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) bne,pn %xcc, .Lmedium_unaligned_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) .Lmedium_noprefetch_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) andncc %o2, 0x20 - 1, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) be,pn %xcc, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) sub %o2, %o5, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) EX_LD(LOAD(ldx, %o1 + 0x10, %g7), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) add %o1, 0x20, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) subcc %o5, 0x20, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) add %o0, 0x20, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) 2: andcc %o2, 0x18, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) be,pt %xcc, 3f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) sub %o2, %o5, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) add %o1, 0x08, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) add %o0, 0x08, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) subcc %o5, 0x08, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) EX_ST(STORE(stx, %o3, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) 3: brz,pt %o2, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) cmp %o2, 0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) bl,pn %xcc, .Ltiny_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) add %o1, 0x04, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) add %o0, 0x04, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) subcc %o2, 0x04, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) bne,pn %xcc, .Ltiny_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) ba,a,pt %xcc, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) .Lmedium_unaligned_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /* First get dest 8 byte aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) sub %g0, %o0, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) and %o3, 0x7, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) brz,pt %o3, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) sub %o2, %o3, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) 1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) add %o1, 1, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) subcc %o3, 1, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) add %o0, 1, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) and %o1, 0x7, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) brz,pn %o3, .Lmedium_noprefetch_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) sll %o3, 3, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) mov 64, %g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) sub %g2, %o3, %g2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) andn %o1, 0x7, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) sllx %o4, %o3, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) andn %o2, 0x08 - 1, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) sub %o2, %o5, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) 1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) add %o1, 0x08, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) subcc %o5, 0x08, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) srlx %g3, %g2, %g7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) or %g7, %o4, %g7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) EX_ST(STORE(stx, %g7, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) add %o0, 0x08, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) sllx %g3, %o3, %o4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) srl %o3, 3, %o3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) add %o1, %o3, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) brz,pn %o2, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) ba,pt %xcc, .Lsmall_unaligned_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) .Ltiny_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) subcc %o2, 1, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) be,pn %xcc, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) EX_ST(STORE(stb, %o3, %o0 + 0x00), memcpy_retl_o2_plus_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) EX_LD(LOAD(ldub, %o1 + 0x01, %o3), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) subcc %o2, 1, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) be,pn %xcc, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) EX_ST(STORE(stb, %o3, %o0 + 0x01), memcpy_retl_o2_plus_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) EX_LD(LOAD(ldub, %o1 + 0x02, %o3), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) ba,pt %xcc, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) EX_ST(STORE(stb, %o3, %o0 + 0x02), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) .Lsmall_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) andcc %g2, 0x3, %g0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) bne,pn %xcc, .Lsmall_unaligned_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) andn %o2, 0x4 - 1, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) sub %o2, %o5, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) add %o1, 0x04, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) subcc %o5, 0x04, %o5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) add %o0, 0x04, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) brz,pt %o2, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) ba,a,pt %xcc, .Ltiny_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) .Lsmall_unaligned_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) 1: EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) add %o1, 1, %o1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) add %o0, 1, %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) subcc %o2, 1, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) bne,pt %xcc, 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) EX_ST(STORE(stb, %o3, %o0 - 0x01), memcpy_retl_o2_plus_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) ba,a,pt %xcc, .Lexit_cp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) .Lsmallrest:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) tst %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) bz,pt %xcc, .Lsmallx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) cmp %o2, 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) blt,pn %xcc, .Lsmallleft3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) sub %o2, 3, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) .Lsmallnotalign4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_3)! read byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) subcc %o2, 4, %o2 ! reduce count by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_7)! write byte & repeat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2_plus_6)! for total of 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) add %o1, 4, %o1 ! advance SRC by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) EX_LD(LOAD(ldub, %o1-2, %o3), memcpy_retl_o2_plus_5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) add %o0, 4, %o0 ! advance DST by 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) EX_ST(STORE(stb, %o3, %o0-2), memcpy_retl_o2_plus_5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) EX_LD(LOAD(ldub, %o1-1, %o3), memcpy_retl_o2_plus_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) bgu,pt %xcc, .Lsmallnotalign4 ! loop til 3 or fewer bytes remain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) EX_ST(STORE(stb, %o3, %o0-1), memcpy_retl_o2_plus_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) addcc %o2, 3, %o2 ! restore count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) bz,pt %xcc, .Lsmallx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) .Lsmallleft3: ! 1, 2, or 3 bytes remain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) subcc %o2, 1, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_1) ! load one byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) bz,pt %xcc, .Lsmallx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_1) ! store one byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2) ! load second byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) subcc %o2, 1, %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) bz,pt %xcc, .Lsmallx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_1)! store second byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) EX_LD(LOAD(ldub, %o1+2, %o3), memcpy_retl_o2) ! load third byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) EX_ST(STORE(stb, %o3, %o0+2), memcpy_retl_o2) ! store third byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) .Lsmallx:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) retl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) mov EX_RETVAL(%g1), %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) .Lsmallfin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) tst %o2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) bnz,pn %xcc, .Lsmallleft3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) nop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) retl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) mov EX_RETVAL(%g1), %o0 ! restore %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) .Lexit_cp:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) retl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) mov EX_RETVAL(%g1), %o0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) .size FUNC_NAME, .-FUNC_NAME
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags