Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <linux/context_tracking.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/nospec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/syscalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/daifflags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/debug-monitors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/fpsimd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/syscall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <asm/unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) long compat_arm_syscall(struct pt_regs *regs, int scno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) long sys_ni_syscall(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) static long do_ni_syscall(struct pt_regs *regs, int scno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	if (is_compat_task()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 		ret = compat_arm_syscall(regs, scno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 		if (ret != -ENOSYS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 			return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	return sys_ni_syscall();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) static long __invoke_syscall(struct pt_regs *regs, syscall_fn_t syscall_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	return syscall_fn(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 			   unsigned int sc_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 			   const syscall_fn_t syscall_table[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	if (scno < sc_nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 		syscall_fn_t syscall_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 		syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 		ret = __invoke_syscall(regs, syscall_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 		ret = do_ni_syscall(regs, scno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	syscall_set_return_value(current, regs, 0, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) static inline bool has_syscall_work(unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	return unlikely(flags & _TIF_SYSCALL_WORK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) int syscall_trace_enter(struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) void syscall_trace_exit(struct pt_regs *regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #ifdef CONFIG_ARM64_ERRATUM_1463225
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) static void cortex_a76_erratum_1463225_svc_handler(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	u32 reg, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	reg = read_sysreg(mdscr_el1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	val = reg | DBG_MDSCR_SS | DBG_MDSCR_KDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	write_sysreg(val, mdscr_el1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	asm volatile("msr daifclr, #8");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	/* We will have taken a single-step exception by this point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	write_sysreg(reg, mdscr_el1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	__this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) static void cortex_a76_erratum_1463225_svc_handler(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) #endif /* CONFIG_ARM64_ERRATUM_1463225 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 			   const syscall_fn_t syscall_table[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	unsigned long flags = current_thread_info()->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	regs->orig_x0 = regs->regs[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	regs->syscallno = scno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	 * BTI note:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	 * The architecture does not guarantee that SPSR.BTYPE is zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	 * on taking an SVC, so we could return to userspace with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	 * non-zero BTYPE after the syscall.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	 * This shouldn't matter except when userspace is explicitly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	 * doing something stupid, such as setting PROT_BTI on a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	 * that lacks conforming BTI/PACIxSP instructions, falling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	 * through from one executable page to another with differing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	 * PROT_BTI, or messing with BTYPE via ptrace: in such cases,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	 * userspace should not be surprised if a SIGILL occurs on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	 * syscall return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	 * So, don't touch regs->pstate & PSR_BTYPE_MASK here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	 * (Similarly for HVC and SMC elsewhere.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	cortex_a76_erratum_1463225_svc_handler();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	local_daif_restore(DAIF_PROCCTX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	if (flags & _TIF_MTE_ASYNC_FAULT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		 * Process the asynchronous tag check fault before the actual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		 * syscall. do_notify_resume() will send a signal to userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		 * before the syscall is restarted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		syscall_set_return_value(current, regs, -ERESTARTNOINTR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	if (has_syscall_work(flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		 * The de-facto standard way to skip a system call using ptrace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		 * is to set the system call to -1 (NO_SYSCALL) and set x0 to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		 * suitable error code for consumption by userspace. However,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		 * this cannot be distinguished from a user-issued syscall(-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		 * and so we must set x0 to -ENOSYS here in case the tracer doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		 * issue the skip and we fall into trace_exit with x0 preserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 		 * This is slightly odd because it also means that if a tracer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		 * sets the system call number to -1 but does not initialise x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		 * then x0 will be preserved for all system calls apart from a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 		 * user-issued syscall(-1). However, requesting a skip and not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		 * setting the return value is unlikely to do anything sensible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		 * anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		if (scno == NO_SYSCALL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 			syscall_set_return_value(current, regs, -ENOSYS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		scno = syscall_trace_enter(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		if (scno == NO_SYSCALL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 			goto trace_exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	invoke_syscall(regs, scno, sc_nr, syscall_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	 * The tracing status may have changed under our feet, so we have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	 * check again. However, if we were tracing entry, then we always trace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	 * exit regardless, as the old entry assembly did.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 		local_daif_mask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		flags = current_thread_info()->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 		if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 		local_daif_restore(DAIF_PROCCTX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) trace_exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	syscall_trace_exit(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static inline void sve_user_discard(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	if (!system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	clear_thread_flag(TIF_SVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	 * task_fpsimd_load() won't be called to update CPACR_EL1 in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	 * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	 * happens if a context switch or kernel_neon_begin() or context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	 * modification (sigreturn, ptrace) intervenes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	 * So, ensure that CPACR_EL1 is already correct for the fast-path case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	sve_user_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) void do_el0_svc(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	sve_user_discard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) void do_el0_svc_compat(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		       compat_sys_call_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #endif