Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * __get_user functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * (C) Copyright 1998 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * (C) Copyright 2005 Andi Kleen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * (C) Copyright 2008 Glauber Costa
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * These functions have a non-standard call interface
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * to make them more efficient, especially as they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * return an error value in addition to the "real"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * return value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * __get_user_X
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * Inputs:	%[r|e]ax contains the address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * Outputs:	%[r|e]ax is error code (0 or -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  *		%[r|e]dx contains zero-extended value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  *		%ecx contains the high half for 32-bit __get_user_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  * These functions should not modify any other registers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  * as they get called from within inline assembly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <asm/page_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <asm/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #include <asm/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #include <asm/smap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #include <asm/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #ifdef CONFIG_X86_5LEVEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #define LOAD_TASK_SIZE_MINUS_N(n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	ALTERNATIVE __stringify(mov $((1 << 47) - 4096 - (n)),%rdx), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 		    __stringify(mov $((1 << 56) - 4096 - (n)),%rdx), X86_FEATURE_LA57
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #define LOAD_TASK_SIZE_MINUS_N(n) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	mov $(TASK_SIZE_MAX - (n)),%_ASM_DX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	.text
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) SYM_FUNC_START(__get_user_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	LOAD_TASK_SIZE_MINUS_N(0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	cmp %_ASM_DX,%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	jae bad_get_user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	and %_ASM_DX, %_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 1:	movzbl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) SYM_FUNC_END(__get_user_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) EXPORT_SYMBOL(__get_user_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) SYM_FUNC_START(__get_user_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	LOAD_TASK_SIZE_MINUS_N(1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	cmp %_ASM_DX,%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	jae bad_get_user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	and %_ASM_DX, %_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 2:	movzwl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) SYM_FUNC_END(__get_user_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) EXPORT_SYMBOL(__get_user_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) SYM_FUNC_START(__get_user_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	LOAD_TASK_SIZE_MINUS_N(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	cmp %_ASM_DX,%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	jae bad_get_user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	and %_ASM_DX, %_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 3:	movl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) SYM_FUNC_END(__get_user_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) EXPORT_SYMBOL(__get_user_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) SYM_FUNC_START(__get_user_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	LOAD_TASK_SIZE_MINUS_N(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	cmp %_ASM_DX,%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	jae bad_get_user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	and %_ASM_DX, %_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 4:	movq (%_ASM_AX),%rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	LOAD_TASK_SIZE_MINUS_N(7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	cmp %_ASM_DX,%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	jae bad_get_user_8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	and %_ASM_DX, %_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 4:	movl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 5:	movl 4(%_ASM_AX),%ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) SYM_FUNC_END(__get_user_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) EXPORT_SYMBOL(__get_user_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* .. and the same for __get_user, just without the range checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) SYM_FUNC_START(__get_user_nocheck_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	ASM_BARRIER_NOSPEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 6:	movzbl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) SYM_FUNC_END(__get_user_nocheck_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) EXPORT_SYMBOL(__get_user_nocheck_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) SYM_FUNC_START(__get_user_nocheck_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	ASM_BARRIER_NOSPEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 7:	movzwl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) SYM_FUNC_END(__get_user_nocheck_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) EXPORT_SYMBOL(__get_user_nocheck_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) SYM_FUNC_START(__get_user_nocheck_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	ASM_BARRIER_NOSPEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 8:	movl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) SYM_FUNC_END(__get_user_nocheck_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) EXPORT_SYMBOL(__get_user_nocheck_4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) SYM_FUNC_START(__get_user_nocheck_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	ASM_STAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	ASM_BARRIER_NOSPEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 9:	movq (%_ASM_AX),%rdx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 9:	movl (%_ASM_AX),%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 10:	movl 4(%_ASM_AX),%ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	xor %eax,%eax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) SYM_FUNC_END(__get_user_nocheck_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) EXPORT_SYMBOL(__get_user_nocheck_8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) bad_get_user:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	xor %edx,%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	mov $(-EFAULT),%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) SYM_CODE_END(.Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) SYM_CODE_START_LOCAL(.Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	ASM_CLAC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) bad_get_user_8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	xor %edx,%edx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	xor %ecx,%ecx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	mov $(-EFAULT),%_ASM_AX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) SYM_CODE_END(.Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* get_user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	_ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	_ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	_ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	_ASM_EXTABLE_UA(4b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	_ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	_ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* __get_user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	_ASM_EXTABLE_UA(6b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	_ASM_EXTABLE_UA(7b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	_ASM_EXTABLE_UA(8b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	_ASM_EXTABLE_UA(9b, .Lbad_get_user_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	_ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	_ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #endif