^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * User address space access functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * The non inlined parts of asm-i386/uaccess.h are here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright 1997 Andi Kleen <ak@muc.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright 1997 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/mmx.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #ifdef CONFIG_X86_INTEL_USERCOPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Alignment at which movsl is preferred for bulk memory copies.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) struct movsl_mask movsl_mask __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #ifdef CONFIG_X86_INTEL_USERCOPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define movsl_is_ok(a1, a2, n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Zero Userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define __do_clear_user(addr,size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) int __d0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) might_fault(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) ASM_STAC "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) "0: rep; stosl\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) " movl %2,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) "1: rep; stosb\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) "2: " ASM_CLAC "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) ".section .fixup,\"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) "3: lea 0(%2,%0,4),%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) " jmp 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) ".previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) _ASM_EXTABLE_UA(0b, 3b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) _ASM_EXTABLE_UA(1b, 2b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) : "=&c"(size), "=&D" (__d0) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * clear_user - Zero a block of memory in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * @to: Destination address, in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * @n: Number of bytes to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Zero a block of memory in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * Return: number of bytes that could not be cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * On success, this will be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) clear_user(void __user *to, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) might_fault();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (access_ok(to, n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) __do_clear_user(to, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) EXPORT_SYMBOL(clear_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * __clear_user - Zero a block of memory in user space, with less checking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * @to: Destination address, in user space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * @n: Number of bytes to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Zero a block of memory in user space. Caller must check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * the specified block with access_ok() before calling this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Return: number of bytes that could not be cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * On success, this will be zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) __clear_user(void __user *to, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) __do_clear_user(to, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) EXPORT_SYMBOL(__clear_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #ifdef CONFIG_X86_INTEL_USERCOPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) __copy_user_intel(void __user *to, const void *from, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int d0, d1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) " .align 2,0x90\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) "1: movl 32(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) " cmpl $67, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) " jbe 3f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) "2: movl 64(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) " .align 2,0x90\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) "3: movl 0(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) "4: movl 4(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) "5: movl %%eax, 0(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) "6: movl %%edx, 4(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) "7: movl 8(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) "8: movl 12(%4),%%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) "9: movl %%eax, 8(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) "10: movl %%edx, 12(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) "11: movl 16(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) "12: movl 20(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) "13: movl %%eax, 16(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) "14: movl %%edx, 20(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) "15: movl 24(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) "16: movl 28(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) "17: movl %%eax, 24(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) "18: movl %%edx, 28(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) "19: movl 32(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) "20: movl 36(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) "21: movl %%eax, 32(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) "22: movl %%edx, 36(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) "23: movl 40(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) "24: movl 44(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) "25: movl %%eax, 40(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "26: movl %%edx, 44(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) "27: movl 48(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) "28: movl 52(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) "29: movl %%eax, 48(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) "30: movl %%edx, 52(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) "31: movl 56(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) "32: movl 60(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) "33: movl %%eax, 56(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) "34: movl %%edx, 60(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) " addl $-64, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) " addl $64, %4\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) " addl $64, %3\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) " cmpl $63, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) " ja 1b\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) "35: movl %0, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) " shrl $2, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) " andl $3, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) " cld\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) "99: rep; movsl\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) "36: movl %%eax, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) "37: rep; movsb\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) "100:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) ".section .fixup,\"ax\"\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) "101: lea 0(%%eax,%0,4),%0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) " jmp 100b\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ".previous\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) _ASM_EXTABLE_UA(1b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) _ASM_EXTABLE_UA(2b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) _ASM_EXTABLE_UA(3b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) _ASM_EXTABLE_UA(4b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) _ASM_EXTABLE_UA(5b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) _ASM_EXTABLE_UA(6b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) _ASM_EXTABLE_UA(7b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) _ASM_EXTABLE_UA(8b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) _ASM_EXTABLE_UA(9b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) _ASM_EXTABLE_UA(10b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) _ASM_EXTABLE_UA(11b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) _ASM_EXTABLE_UA(12b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) _ASM_EXTABLE_UA(13b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) _ASM_EXTABLE_UA(14b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) _ASM_EXTABLE_UA(15b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) _ASM_EXTABLE_UA(16b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) _ASM_EXTABLE_UA(17b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) _ASM_EXTABLE_UA(18b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) _ASM_EXTABLE_UA(19b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) _ASM_EXTABLE_UA(20b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) _ASM_EXTABLE_UA(21b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) _ASM_EXTABLE_UA(22b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) _ASM_EXTABLE_UA(23b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) _ASM_EXTABLE_UA(24b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) _ASM_EXTABLE_UA(25b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) _ASM_EXTABLE_UA(26b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) _ASM_EXTABLE_UA(27b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) _ASM_EXTABLE_UA(28b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) _ASM_EXTABLE_UA(29b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) _ASM_EXTABLE_UA(30b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) _ASM_EXTABLE_UA(31b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) _ASM_EXTABLE_UA(32b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) _ASM_EXTABLE_UA(33b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) _ASM_EXTABLE_UA(34b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) _ASM_EXTABLE_UA(35b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) _ASM_EXTABLE_UA(36b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) _ASM_EXTABLE_UA(37b, 100b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) _ASM_EXTABLE_UA(99b, 101b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) : "=&c"(size), "=&D" (d0), "=&S" (d1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) : "1"(to), "2"(from), "0"(size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) : "eax", "edx", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static unsigned long __copy_user_intel_nocache(void *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) const void __user *from, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) int d0, d1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) " .align 2,0x90\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) "0: movl 32(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) " cmpl $67, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) " jbe 2f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) "1: movl 64(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) " .align 2,0x90\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) "2: movl 0(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) "21: movl 4(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) " movnti %%eax, 0(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) " movnti %%edx, 4(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) "3: movl 8(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) "31: movl 12(%4),%%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) " movnti %%eax, 8(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) " movnti %%edx, 12(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) "4: movl 16(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) "41: movl 20(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) " movnti %%eax, 16(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) " movnti %%edx, 20(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) "10: movl 24(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) "51: movl 28(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) " movnti %%eax, 24(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) " movnti %%edx, 28(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) "11: movl 32(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) "61: movl 36(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) " movnti %%eax, 32(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) " movnti %%edx, 36(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) "12: movl 40(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) "71: movl 44(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) " movnti %%eax, 40(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) " movnti %%edx, 44(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) "13: movl 48(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) "81: movl 52(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) " movnti %%eax, 48(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) " movnti %%edx, 52(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) "14: movl 56(%4), %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) "91: movl 60(%4), %%edx\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) " movnti %%eax, 56(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) " movnti %%edx, 60(%3)\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) " addl $-64, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) " addl $64, %4\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) " addl $64, %3\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) " cmpl $63, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) " ja 0b\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) " sfence \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) "5: movl %0, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) " shrl $2, %0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) " andl $3, %%eax\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) " cld\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) "6: rep; movsl\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) " movl %%eax,%0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) "7: rep; movsb\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) "8:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) ".section .fixup,\"ax\"\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) "9: lea 0(%%eax,%0,4),%0\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) "16: jmp 8b\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) ".previous\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) _ASM_EXTABLE_UA(0b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) _ASM_EXTABLE_UA(1b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) _ASM_EXTABLE_UA(2b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) _ASM_EXTABLE_UA(21b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) _ASM_EXTABLE_UA(3b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) _ASM_EXTABLE_UA(31b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) _ASM_EXTABLE_UA(4b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) _ASM_EXTABLE_UA(41b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) _ASM_EXTABLE_UA(10b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) _ASM_EXTABLE_UA(51b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) _ASM_EXTABLE_UA(11b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) _ASM_EXTABLE_UA(61b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) _ASM_EXTABLE_UA(12b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) _ASM_EXTABLE_UA(71b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) _ASM_EXTABLE_UA(13b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) _ASM_EXTABLE_UA(81b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) _ASM_EXTABLE_UA(14b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) _ASM_EXTABLE_UA(91b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) _ASM_EXTABLE_UA(6b, 9b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) _ASM_EXTABLE_UA(7b, 16b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) : "=&c"(size), "=&D" (d0), "=&S" (d1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) : "1"(to), "2"(from), "0"(size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) : "eax", "edx", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * Leave these declared but undefined. They should not be any references to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long __copy_user_intel(void __user *to, const void *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) unsigned long size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #endif /* CONFIG_X86_INTEL_USERCOPY */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /* Generic arbitrary sized copy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #define __copy_user(to, from, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) int __d0, __d1, __d2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) __asm__ __volatile__( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) " cmp $7,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) " jbe 1f\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) " movl %1,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) " negl %0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) " andl $7,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) " subl %0,%3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) "4: rep; movsb\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) " movl %3,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) " shrl $2,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) " andl $3,%3\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) " .align 2,0x90\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) "0: rep; movsl\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) " movl %3,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) "1: rep; movsb\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) "2:\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) ".section .fixup,\"ax\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) "5: addl %3,%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) " jmp 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) "3: lea 0(%3,%0,4),%0\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) " jmp 2b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) ".previous\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) _ASM_EXTABLE_UA(4b, 5b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) _ASM_EXTABLE_UA(0b, 3b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) _ASM_EXTABLE_UA(1b, 2b) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) : "3"(size), "0"(size), "1"(to), "2"(from) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) __uaccess_begin_nospec();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (movsl_is_ok(to, from, n))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) __copy_user(to, from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) n = __copy_user_intel(to, from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) __uaccess_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) EXPORT_SYMBOL(__copy_user_ll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) unsigned long n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) __uaccess_begin_nospec();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #ifdef CONFIG_X86_INTEL_USERCOPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) n = __copy_user_intel_nocache(to, from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) __copy_user(to, from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) __copy_user(to, from, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) __uaccess_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags