^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* Extracted from GLIBC memcpy.c and memcopy.h, which is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) This file is part of the GNU C Library.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) Contributed by Torbjorn Granlund (tege@sics.se).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) The GNU C Library is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) modify it under the terms of the GNU Lesser General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) License as published by the Free Software Foundation; either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) version 2.1 of the License, or (at your option) any later version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) The GNU C Library is distributed in the hope that it will be useful,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) but WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) Lesser General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) You should have received a copy of the GNU Lesser General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) License along with the GNU C Library; if not, see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) <http://www.gnu.org/licenses/>. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) /* Type to use for aligned memory operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) This should normally be the biggest type supported by a single load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) and store. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define op_t unsigned long int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define OPSIZ (sizeof(op_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* Optimal type for storing bytes in registers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define reg_char char
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define MERGE(w0, sh_1, w1, sh_2) (((w0) >> (sh_1)) | ((w1) << (sh_2)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* Copy exactly NBYTES bytes from SRC_BP to DST_BP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) without any assumptions about alignment of the pointers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) size_t __nbytes = (nbytes); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) while (__nbytes > 0) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unsigned char __x = ((unsigned char *) src_bp)[0]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) src_bp += 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) __nbytes -= 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) ((unsigned char *) dst_bp)[0] = __x; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) dst_bp += 1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* Copy *up to* NBYTES bytes from SRC_BP to DST_BP, with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) the assumption that DST_BP is aligned on an OPSIZ multiple. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) not all bytes could be easily copied, store remaining number of bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) in NBYTES_LEFT, otherwise store 0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* extern void _wordcopy_fwd_aligned __P ((long int, long int, size_t)); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* extern void _wordcopy_fwd_dest_aligned __P ((long int, long int, size_t)); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (src_bp % OPSIZ == 0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) _wordcopy_fwd_aligned(dst_bp, src_bp, (nbytes) / OPSIZ);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) else \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) _wordcopy_fwd_dest_aligned(dst_bp, src_bp, (nbytes) / OPSIZ);\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) src_bp += (nbytes) & -OPSIZ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) dst_bp += (nbytes) & -OPSIZ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) (nbytes_left) = (nbytes) % OPSIZ; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* Threshold value for when to enter the unrolled loops. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define OP_T_THRES 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* _wordcopy_fwd_aligned -- Copy block beginning at SRCP to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) block beginning at DSTP with LEN `op_t' words (not LEN bytes!).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) Both SRCP and DSTP should be aligned for memory operations on `op_t's. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* stream-lined (read x8 + write x8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static void _wordcopy_fwd_aligned(long int dstp, long int srcp, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) while (len > 7) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) register op_t a0, a1, a2, a3, a4, a5, a6, a7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) a0 = ((op_t *) srcp)[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) a1 = ((op_t *) srcp)[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) a2 = ((op_t *) srcp)[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) a3 = ((op_t *) srcp)[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) a4 = ((op_t *) srcp)[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) a5 = ((op_t *) srcp)[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) a6 = ((op_t *) srcp)[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) a7 = ((op_t *) srcp)[7];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) ((op_t *) dstp)[0] = a0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) ((op_t *) dstp)[1] = a1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) ((op_t *) dstp)[2] = a2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) ((op_t *) dstp)[3] = a3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) ((op_t *) dstp)[4] = a4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) ((op_t *) dstp)[5] = a5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ((op_t *) dstp)[6] = a6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) ((op_t *) dstp)[7] = a7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) srcp += 8 * OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) dstp += 8 * OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) len -= 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) while (len > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *(op_t *)dstp = *(op_t *)srcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) srcp += OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) dstp += OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) len -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* _wordcopy_fwd_dest_aligned -- Copy block beginning at SRCP to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) block beginning at DSTP with LEN `op_t' words (not LEN bytes!).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) DSTP should be aligned for memory operations on `op_t's, but SRCP must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) *not* be aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* stream-lined (read x4 + write x4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static void _wordcopy_fwd_dest_aligned(long int dstp, long int srcp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) op_t ap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) int sh_1, sh_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /* Calculate how to shift a word read at the memory operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) aligned srcp to make it aligned for copy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) sh_1 = 8 * (srcp % OPSIZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) sh_2 = 8 * OPSIZ - sh_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Make SRCP aligned by rounding it down to the beginning of the `op_t'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) it points in the middle of. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) srcp &= -OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) ap = ((op_t *) srcp)[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) srcp += OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) while (len > 3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) op_t a0, a1, a2, a3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) a0 = ((op_t *) srcp)[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) a1 = ((op_t *) srcp)[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) a2 = ((op_t *) srcp)[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) a3 = ((op_t *) srcp)[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) ((op_t *) dstp)[0] = MERGE(ap, sh_1, a0, sh_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) ((op_t *) dstp)[1] = MERGE(a0, sh_1, a1, sh_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) ((op_t *) dstp)[2] = MERGE(a1, sh_1, a2, sh_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) ((op_t *) dstp)[3] = MERGE(a2, sh_1, a3, sh_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) ap = a3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) srcp += 4 * OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) dstp += 4 * OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) len -= 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) while (len > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) register op_t a0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) a0 = ((op_t *) srcp)[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) ((op_t *) dstp)[0] = MERGE(ap, sh_1, a0, sh_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) ap = a0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) srcp += OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) dstp += OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) len -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) void *memcpy(void *dstpp, const void *srcpp, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned long int dstp = (long int) dstpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) unsigned long int srcp = (long int) srcpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* Copy from the beginning to the end. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /* If there not too few bytes to copy, use word copy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (len >= OP_T_THRES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /* Copy just a few bytes to make DSTP aligned. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) len -= (-dstp) % OPSIZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* Copy whole pages from SRCP to DSTP by virtual address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) manipulation, as much as possible. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /* PAGE_COPY_FWD_MAYBE (dstp, srcp, len, len); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* Copy from SRCP to DSTP taking advantage of the known
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) alignment of DSTP. Number of bytes remaining is put in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) third argument, i.e. in LEN. This number may vary from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) machine to machine. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) WORD_COPY_FWD(dstp, srcp, len, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* Fall out and copy the tail. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /* There are just a few bytes to copy. Use byte memory operations. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) BYTE_COPY_FWD(dstp, srcp, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return dstpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) void *memcpyb(void *dstpp, const void *srcpp, unsigned len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) unsigned long int dstp = (long int) dstpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) unsigned long int srcp = (long int) srcpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) BYTE_COPY_FWD(dstp, srcp, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return dstpp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags