^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * vineetg: June 2010
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * -__clear_user( ) called multiple times during elf load was byte loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * converted to do as much word clear as possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * vineetg: Dec 2009
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * -Hand crafted constant propagation for "constant" copy sizes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * -stock kernel shrunk by 33K at -O3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * vineetg: Sept 2009
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * -Added option to (UN)inline copy_(to|from)_user to reduce code sz
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * -kernel shrunk by 200K even at -O3 (gcc 4.2.1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * -Enabled when doing -Os
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #ifndef _ASM_ARC_UACCESS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define _ASM_ARC_UACCESS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/string.h> /* for generic string functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define __kernel_ok (uaccess_kernel())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * Algorithmically, for __user_ok() we want do:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * (start < TASK_SIZE) && (start+len < TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * where TASK_SIZE could either be retrieved from thread_info->addr_limit or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * emitted directly in code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * This can however be rewritten as follows:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * (len <= TASK_SIZE) && (start+len < TASK_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Because it essentially checks if buffer end is within limit and @len is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * non-ngeative, which implies that buffer start will be within limit too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * The reason for rewriting being, for majority of cases, @len is generally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * compile time constant, causing first sub-expression to be compile time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * subsumed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * The second part would generate weird large LIMMs e.g. (0x6000_0000 - 0x10),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * so we check for TASK_SIZE using get_fs() since the addr_limit load from mem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * would already have been done at this call site for __kernel_ok()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define __user_ok(addr, sz) (((sz) <= TASK_SIZE) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) ((addr) <= (get_fs() - (sz))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define __access_ok(addr, sz) (unlikely(__kernel_ok) || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) likely(__user_ok((addr), (sz))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /*********** Single byte/hword/word copies ******************/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define __get_user_fn(sz, u, k) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) long __ret = 0; /* success by default */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) switch (sz) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) case 1: __arc_get_user_one(*(k), u, "ldb", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) case 2: __arc_get_user_one(*(k), u, "ldw", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) case 4: __arc_get_user_one(*(k), u, "ld", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) case 8: __arc_get_user_one_64(*(k), u, __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) __ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Returns 0 on success, -EFAULT if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * @ret already contains 0 - given that errors will be less likely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * (hence +r asm constraint below).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * In case of error, fixup code will make it -EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define __arc_get_user_one(dst, src, op, ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) "1: "op" %1,[%2]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) "2: ;nop\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) " .section .fixup, \"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) "3: # return -EFAULT\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) " mov %0, %3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) " # zero out dst ptr\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) " mov %1, 0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) " j 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) " .section __ex_table, \"a\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) " .word 1b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) : "+r" (ret), "=r" (dst) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) : "r" (src), "ir" (-EFAULT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define __arc_get_user_one_64(dst, src, ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) "1: ld %1,[%2]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) "4: ld %R1,[%2, 4]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) "2: ;nop\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) " .section .fixup, \"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) "3: # return -EFAULT\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) " mov %0, %3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) " # zero out dst ptr\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) " mov %1, 0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) " mov %R1, 0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) " j 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) " .section __ex_table, \"a\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) " .word 1b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) " .word 4b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) : "+r" (ret), "=r" (dst) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) : "r" (src), "ir" (-EFAULT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #define __put_user_fn(sz, u, k) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) long __ret = 0; /* success by default */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) switch (sz) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) case 1: __arc_put_user_one(*(k), u, "stb", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) case 2: __arc_put_user_one(*(k), u, "stw", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) case 4: __arc_put_user_one(*(k), u, "st", __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) case 8: __arc_put_user_one_64(*(k), u, __ret); break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) __ret; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define __arc_put_user_one(src, dst, op, ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) "1: "op" %1,[%2]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) "2: ;nop\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) " .section .fixup, \"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) "3: mov %0, %3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) " j 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) " .section __ex_table, \"a\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) " .word 1b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) : "+r" (ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) : "r" (src), "r" (dst), "ir" (-EFAULT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #define __arc_put_user_one_64(src, dst, ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) "1: st %1,[%2]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) "4: st %R1,[%2, 4]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) "2: ;nop\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) " .section .fixup, \"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) "3: mov %0, %3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) " j 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) " .section __ex_table, \"a\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) " .align 4\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) " .word 1b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) " .word 4b,3b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) " .previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) : "+r" (ret) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) : "r" (src), "r" (dst), "ir" (-EFAULT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) raw_copy_from_user(void *to, const void __user *from, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) long res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) char val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned long tmp1, tmp2, tmp3, tmp4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unsigned long orig_n = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (n == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* unaligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned char tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) __asm__ __volatile__ (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) " mov.f lp_count, %0 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) " lpnz 2f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) "1: ldb.ab %1, [%3, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) " stb.ab %1, [%2, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) " sub %0,%0,1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) "2: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) "3: j 2b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) " .word 1b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) : "+r" (n),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * Note as an '&' earlyclobber operand to make sure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * temporary register inside the loop is not the same as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * FROM or TO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) "=&r" (tmp), "+r" (to), "+r" (from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Hand-crafted constant propagation to reduce code sz of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * laddered copy 16x,8,4,2,1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (__builtin_constant_p(orig_n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) res = orig_n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (orig_n / 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) orig_n = orig_n % 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) " lsr lp_count, %7,4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) " lp 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) "1: ld.ab %3, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) "11: ld.ab %4, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) "12: ld.ab %5, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) "13: ld.ab %6, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) " st.ab %3, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) " st.ab %4, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) " st.ab %5, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) " st.ab %6, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) " sub %0,%0,16 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) "3: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) "4: j 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) " .word 11b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) " .word 12b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) " .word 13b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) : "+r" (res), "+r"(to), "+r"(from),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) : "ir"(n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (orig_n / 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) orig_n = orig_n % 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) "14: ld.ab %3, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) "15: ld.ab %4, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) " st.ab %3, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) " st.ab %4, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) " sub %0,%0,8 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) "31: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) "4: j 31b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) " .word 14b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) " .word 15b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) : "+r" (res), "+r"(to), "+r"(from),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) "=r"(tmp1), "=r"(tmp2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (orig_n / 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) orig_n = orig_n % 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) "16: ld.ab %3, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) " st.ab %3, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) " sub %0,%0,4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) "32: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) "4: j 32b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) " .word 16b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (orig_n / 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) orig_n = orig_n % 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) "17: ldw.ab %3, [%2,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) " stw.ab %3, [%1,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) " sub %0,%0,2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) "33: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) "4: j 33b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) " .word 17b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (orig_n & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) "18: ldb.ab %3, [%2,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) " stb.ab %3, [%1,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) " sub %0,%0,1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) "34: ; nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) "4: j 34b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) " .word 18b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) " mov %0,%3 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) " lsr.f lp_count, %3,4 \n" /* 16x bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) " lpnz 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) "1: ld.ab %5, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) "11: ld.ab %6, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) "12: ld.ab %7, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) "13: ld.ab %8, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) " st.ab %5, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) " st.ab %6, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) " st.ab %7, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) " st.ab %8, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) " sub %0,%0,16 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) "3: and.f %3,%3,0xf \n" /* stragglers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) " bz 34f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) " bbit0 %3,3,31f \n" /* 8 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) "14: ld.ab %5, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) "15: ld.ab %6, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) " st.ab %5, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) " st.ab %6, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) " sub.f %0,%0,8 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) "31: bbit0 %3,2,32f \n" /* 4 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) "16: ld.ab %5, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) " st.ab %5, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) " sub.f %0,%0,4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) "32: bbit0 %3,1,33f \n" /* 2 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) "17: ldw.ab %5, [%2,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) " stw.ab %5, [%1,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) " sub.f %0,%0,2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) "33: bbit0 %3,0,34f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) "18: ldb.ab %5, [%2,1] \n" /* 1 byte left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) " stb.ab %5, [%1,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) " sub.f %0,%0,1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) "34: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) "4: j 34b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) " .word 11b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) " .word 12b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) " .word 13b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) " .word 14b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) " .word 15b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) " .word 16b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) " .word 17b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) " .word 18b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static inline unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) raw_copy_to_user(void __user *to, const void *from, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) long res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) char val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) unsigned long tmp1, tmp2, tmp3, tmp4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) unsigned long orig_n = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (n == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* unaligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) unsigned char tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) " mov.f lp_count, %0 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) " lpnz 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) " ldb.ab %1, [%3, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) "1: stb.ab %1, [%2, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) " sub %0, %0, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) "3: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) "4: j 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) : "+r" (n),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /* Note as an '&' earlyclobber operand to make sure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * temporary register inside the loop is not the same as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * FROM or TO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) "=&r" (tmp), "+r" (to), "+r" (from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (__builtin_constant_p(orig_n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) res = orig_n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (orig_n / 16) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) orig_n = orig_n % 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) " lsr lp_count, %7,4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) " lp 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) " ld.ab %3, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) " ld.ab %4, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) " ld.ab %5, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) " ld.ab %6, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) "1: st.ab %3, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) "11: st.ab %4, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) "12: st.ab %5, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) "13: st.ab %6, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) " sub %0, %0, 16 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) "3:;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) "4: j 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) " .word 11b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) " .word 12b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) " .word 13b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) : "+r" (res), "+r"(to), "+r"(from),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) : "ir"(n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (orig_n / 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) orig_n = orig_n % 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) " ld.ab %3, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) " ld.ab %4, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) "14: st.ab %3, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) "15: st.ab %4, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) " sub %0, %0, 8 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) "31:;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) "4: j 31b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) " .word 14b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) " .word 15b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) : "+r" (res), "+r"(to), "+r"(from),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) "=r"(tmp1), "=r"(tmp2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (orig_n / 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) orig_n = orig_n % 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) " ld.ab %3, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) "16: st.ab %3, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) " sub %0, %0, 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) "32:;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) "4: j 32b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) " .word 16b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (orig_n / 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) orig_n = orig_n % 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) " ldw.ab %3, [%2,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) "17: stw.ab %3, [%1,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) " sub %0, %0, 2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) "33:;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) "4: j 33b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) " .word 17b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (orig_n & 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) " ldb.ab %3, [%2,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) "18: stb.ab %3, [%1,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) " sub %0, %0, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) "34: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) "4: j 34b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) " .word 18b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) : "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) " mov %0,%3 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) " lsr.f lp_count, %3,4 \n" /* 16x bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) " lpnz 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) " ld.ab %5, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) " ld.ab %6, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) " ld.ab %7, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) " ld.ab %8, [%2, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) "1: st.ab %5, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) "11: st.ab %6, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) "12: st.ab %7, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) "13: st.ab %8, [%1, 4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) " sub %0, %0, 16 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) "3: and.f %3,%3,0xf \n" /* stragglers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) " bz 34f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) " bbit0 %3,3,31f \n" /* 8 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) " ld.ab %5, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) " ld.ab %6, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) "14: st.ab %5, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) "15: st.ab %6, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) " sub.f %0, %0, 8 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) "31: bbit0 %3,2,32f \n" /* 4 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) " ld.ab %5, [%2,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) "16: st.ab %5, [%1,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) " sub.f %0, %0, 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) "32: bbit0 %3,1,33f \n" /* 2 bytes left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) " ldw.ab %5, [%2,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) "17: stw.ab %5, [%1,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) " sub.f %0, %0, 2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) "33: bbit0 %3,0,34f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) " ldb.ab %5, [%2,1] \n" /* 1 byte left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) "18: stb.ab %5, [%1,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) " sub.f %0, %0, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) "34: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) "4: j 34b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) " .word 11b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) " .word 12b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) " .word 13b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) " .word 14b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) " .word 15b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) " .word 16b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) " .word 17b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) " .word 18b,4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) : "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) "=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) static inline unsigned long __arc_clear_user(void __user *to, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) long res = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) unsigned char *d_char = to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) " bbit0 %0, 0, 1f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) "75: stb.ab %2, [%0,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) " sub %1, %1, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) "1: bbit0 %0, 1, 2f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) "76: stw.ab %2, [%0,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) " sub %1, %1, 2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) "2: asr.f lp_count, %1, 2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) " lpnz 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) "77: st.ab %2, [%0,4] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) " sub %1, %1, 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) "3: bbit0 %1, 1, 4f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) "78: stw.ab %2, [%0,2] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) " sub %1, %1, 2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) "4: bbit0 %1, 0, 5f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) "79: stb.ab %2, [%0,1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) " sub %1, %1, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) "5: \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) "3: j 5b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) " .word 75b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) " .word 76b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) " .word 77b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) " .word 78b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) " .word 79b, 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) : "+r"(d_char), "+r"(res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) : "i"(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) static inline long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) __arc_strncpy_from_user(char *dst, const char __user *src, long count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) long res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) char val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (count == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) " mov lp_count, %5 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) " lp 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) "1: ldb.ab %3, [%2, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) " breq.d %3, 0, 3f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) " stb.ab %3, [%1, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) " add %0, %0, 1 # Num of NON NULL bytes copied \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) "3: \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) "4: mov %0, %4 # sets @res as -EFAULT \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) " j 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) : "+r"(res), "+r"(dst), "+r"(src), "=r"(val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) : "g"(-EFAULT), "r"(count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) : "lp_count", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static inline long __arc_strnlen_user(const char __user *s, long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) long res, tmp1, cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) char val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) " mov %2, %1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) "1: ldb.ab %3, [%0, 1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) " breq.d %3, 0, 2f \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) " sub.f %2, %2, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) " sub %2, %2, 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) "2: sub %0, %1, %2 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) "3: ;nop \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) " .section .fixup, \"ax\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) "4: mov %0, 0 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) " j 3b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) " .section __ex_table, \"a\" \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) " .align 4 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) " .word 1b, 4b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) " .previous \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) : "=r"(res), "=r"(tmp1), "=r"(cnt), "=r"(val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) : "0"(s), "1"(n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) #ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #define INLINE_COPY_TO_USER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) #define INLINE_COPY_FROM_USER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) #define __clear_user(d, n) __arc_clear_user(d, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) #define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) #define __strnlen_user(s, n) __arc_strnlen_user(s, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) extern unsigned long arc_clear_user_noinline(void __user *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) unsigned long n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) extern long arc_strncpy_from_user_noinline (char *dst, const char __user *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) long count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) extern long arc_strnlen_user_noinline(const char __user *src, long n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) #define __clear_user(d, n) arc_clear_user_noinline(d, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) #define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) #define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) #include <asm/segment.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) #include <asm-generic/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags