^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Based on arch/arm/kernel/process.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Original Copyright (C) 1995 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 1996-2000 Russell King - Converted to ARM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2012 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <stdarg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/efi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/elf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/sched/task.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/sched/task_stack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/lockdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/nospec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/stddef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/user.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/reboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/elfcore.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/tick.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/utsname.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/hw_breakpoint.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/personality.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <trace/events/power.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/prctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <trace/hooks/fpsimd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <asm/arch_gicv3.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <asm/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <asm/exec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <asm/fpsimd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <asm/mte.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <asm/pointer_auth.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <asm/stacktrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <linux/stackprotector.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned long __stack_chk_guard __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) EXPORT_SYMBOL(__stack_chk_guard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * Function pointers to optional machine specific functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) void (*pm_power_off)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) EXPORT_SYMBOL_GPL(pm_power_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static void noinstr __cpu_do_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) dsb(sy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) wfi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static void noinstr __cpu_do_idle_irqprio(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unsigned long pmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) unsigned long daif_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) daif_bits = read_sysreg(daif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) write_sysreg(daif_bits | PSR_I_BIT, daif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * Unmask PMR before going idle to make sure interrupts can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * be raised.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pmr = gic_read_pmr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __cpu_do_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) gic_write_pmr(pmr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) write_sysreg(daif_bits, daif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * cpu_do_idle()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * Idle the processor (wait for interrupt).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * If the CPU supports priority masking we must do additional work to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * ensure that interrupts are not masked at the PMR (because the core will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * not wake up if we block the wake up signal in the interrupt controller).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) void noinstr cpu_do_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (system_uses_irq_prio_masking())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) __cpu_do_idle_irqprio();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) __cpu_do_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * This is our default idle handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void noinstr arch_cpu_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * This should do all the clock switching and wait for interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * tricks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) cpu_do_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) raw_local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) void arch_cpu_idle_enter(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) idle_notifier_call_chain(IDLE_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) void arch_cpu_idle_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) idle_notifier_call_chain(IDLE_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) void arch_cpu_idle_dead(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) cpu_die();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * Called by kexec, immediately prior to machine_kexec().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * This must completely disable all secondary CPUs; simply causing those CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * kexec'd kernel to use any and all RAM as it sees fit, without having to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) void machine_shutdown(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) smp_shutdown_nonboot_cpus(reboot_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * Halting simply requires that the secondary CPUs stop performing any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * activity (executing tasks, handling interrupts). smp_send_stop()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * achieves this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) void machine_halt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) smp_send_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * Power-off simply requires that the secondary CPUs stop performing any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * activity (executing tasks, handling interrupts). smp_send_stop()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * achieves this. When the system power is turned off, it will take all CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * with it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void machine_power_off(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) smp_send_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (pm_power_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pm_power_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Restart requires that the secondary CPUs stop performing any activity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * while the primary CPU resets the system. Systems with multiple CPUs must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * provide a HW restart implementation, to ensure that all CPUs reset at once.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * This is required so that any code running after reset on the primary CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * doesn't have to co-ordinate with other CPUs to ensure they aren't still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * executing pre-reset code, and using RAM that the primary CPU's code wishes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * to use. Implementing such co-ordination would be essentially impossible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) void machine_restart(char *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* Disable interrupts first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) smp_send_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) do_kernel_pre_restart(cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * UpdateCapsule() depends on the system being reset via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * ResetSystem().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (efi_enabled(EFI_RUNTIME_SERVICES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) efi_reboot(reboot_mode, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /* Now call the architecture specific reboot code. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) do_kernel_restart(cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * Whoops - the architecture was unable to reboot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) printk("Reboot failed -- System halted\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static const char *const btypes[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) bstr(NONE, "--"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) bstr( JC, "jc"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) bstr( C, "-c"),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) bstr( J , "j-")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #undef bstr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static void print_pstate(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) u64 pstate = regs->pstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (compat_user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) pstate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) pstate & PSR_AA32_N_BIT ? 'N' : 'n',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) pstate & PSR_AA32_Z_BIT ? 'Z' : 'z',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) pstate & PSR_AA32_C_BIT ? 'C' : 'c',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) pstate & PSR_AA32_V_BIT ? 'V' : 'v',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) pstate & PSR_AA32_Q_BIT ? 'Q' : 'q',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) pstate & PSR_AA32_T_BIT ? "T32" : "A32",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pstate & PSR_AA32_E_BIT ? "BE" : "LE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) pstate & PSR_AA32_A_BIT ? 'A' : 'a',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) pstate & PSR_AA32_I_BIT ? 'I' : 'i',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) pstate & PSR_AA32_F_BIT ? 'F' : 'f');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) PSR_BTYPE_SHIFT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO BTYPE=%s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) pstate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) pstate & PSR_N_BIT ? 'N' : 'n',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) pstate & PSR_Z_BIT ? 'Z' : 'z',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) pstate & PSR_C_BIT ? 'C' : 'c',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) pstate & PSR_V_BIT ? 'V' : 'v',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pstate & PSR_D_BIT ? 'D' : 'd',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pstate & PSR_A_BIT ? 'A' : 'a',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) pstate & PSR_I_BIT ? 'I' : 'i',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) pstate & PSR_F_BIT ? 'F' : 'f',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) pstate & PSR_PAN_BIT ? '+' : '-',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) pstate & PSR_UAO_BIT ? '+' : '-',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) pstate & PSR_TCO_BIT ? '+' : '-',
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) btype_str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * dump a block of kernel memory from around the given address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static void show_data(unsigned long addr, int nbytes, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) int nlines;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) u32 *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * don't attempt to dump non-kernel addresses or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * values that are probably just small negative numbers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (addr < PAGE_OFFSET || addr > -4096UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) printk("\n%s: %#lx:\n", name, addr + nbytes / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * round address down to a 32 bit boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * and always dump a multiple of 32 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) p = (u32 *)(addr & ~(sizeof(u32) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) nbytes += (addr & (sizeof(u32) - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) nlines = (nbytes + 31) / 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) for (i = 0; i < nlines; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * just display low 16 bits of address to keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * each line of the dump < 80 characters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (i == (nlines / 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) printk("%04lx*", (unsigned long)p & 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) printk("%04lx ", (unsigned long)p & 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) for (j = 0; j < 8; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) u32 data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (aarch64_insn_read((void *)p, &data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) pr_cont(" ********");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) pr_cont(" %08x", data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) ++p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static void show_extra_register_data(struct pt_regs *regs, int nbytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) mm_segment_t fs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) fs = get_fs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) set_fs(KERNEL_DS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) show_data(regs->pc - nbytes, nbytes * 2, "PC");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) show_data(regs->regs[30] - nbytes, nbytes * 2, "LR");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) show_data(regs->sp - nbytes, nbytes * 2, "SP");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) for (i = 0; i < 30; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) char name[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) snprintf(name, sizeof(name), "X%u", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) show_data(regs->regs[i] - nbytes, nbytes * 2, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) set_fs(fs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) void __show_regs(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) int i, top_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) u64 lr, sp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (compat_user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) lr = regs->compat_lr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) sp = regs->compat_sp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) top_reg = 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) lr = regs->regs[30];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) sp = regs->sp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) top_reg = 29;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) show_regs_print_info(KERN_DEFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) print_pstate(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (!user_mode(regs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) printk("pc : %pS\n", (void *)regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) printk("pc : %016llx\n", regs->pc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) printk("lr : %016llx\n", lr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) printk("sp : %016llx\n", sp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (system_uses_irq_prio_masking())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) printk("pmr_save: %08llx\n", regs->pmr_save);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) i = top_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) while (i >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) printk("x%-2d: %016llx ", i, regs->regs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (i % 2 == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) pr_cont("x%-2d: %016llx ", i, regs->regs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) void show_regs(struct pt_regs * regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) __show_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) dump_backtrace(regs, NULL, KERN_DEFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (!user_mode(regs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) show_extra_register_data(regs, 512);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) EXPORT_SYMBOL_GPL(show_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) static void tls_thread_flush(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) write_sysreg(0, tpidr_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (is_compat_task()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) current->thread.uw.tp_value = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * We need to ensure ordering between the shadow state and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * hardware state, so that we don't corrupt the hardware state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * with a stale shadow state during context switch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) write_sysreg(0, tpidrro_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static void flush_tagged_addr_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) clear_thread_flag(TIF_TAGGED_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) void flush_thread(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) fpsimd_flush_thread();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) tls_thread_flush();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) flush_ptrace_hw_breakpoint(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) flush_tagged_addr_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) void release_thread(struct task_struct *dead_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) void arch_release_task_struct(struct task_struct *tsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) fpsimd_release_task(tsk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (current->mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) fpsimd_preserve_current_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) *dst = *src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) /* We rely on the above assignment to initialize dst's thread_flags: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * Detach src's sve_state (if any) from dst so that it does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * get erroneously used or freed prematurely. dst's sve_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * will be allocated on demand later on if dst uses SVE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * For consistency, also clear TIF_SVE here: this could be done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * later in copy_process(), but to avoid tripping up future
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * maintainers it is best not to leave TIF_SVE and sve_state in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * an inconsistent state, even temporarily.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) dst->thread.sve_state = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) clear_tsk_thread_flag(dst, TIF_SVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /* clear any pending asynchronous tag fault raised by the parent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) asmlinkage void ret_from_fork(void) asm("ret_from_fork");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int copy_thread(unsigned long clone_flags, unsigned long stack_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) unsigned long stk_sz, struct task_struct *p, unsigned long tls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) struct pt_regs *childregs = task_pt_regs(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * In case p was allocated the same task_struct pointer as some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * other recently-exited task, make sure p is disassociated from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * any cpu that may have run that now-exited task recently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * Otherwise we could erroneously skip reloading the FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * registers for p.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) fpsimd_flush_task_state(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ptrauth_thread_init_kernel(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (likely(!(p->flags & PF_KTHREAD))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) *childregs = *current_pt_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) childregs->regs[0] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * Read the current TLS pointer from tpidr_el0 as it may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * out-of-sync with the saved value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) *task_user_tls(p) = read_sysreg(tpidr_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (stack_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (is_compat_thread(task_thread_info(p)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) childregs->compat_sp = stack_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) childregs->sp = stack_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * If a TLS pointer was passed to clone, use it for the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (clone_flags & CLONE_SETTLS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) p->thread.uw.tp_value = tls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * A kthread has no context to ERET to, so ensure any buggy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * ERET is treated as an illegal exception return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * When a user task is created from a kthread, childregs will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * be initialized by start_thread() or start_compat_thread().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) memset(childregs, 0, sizeof(struct pt_regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) p->thread.cpu_context.x19 = stack_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) p->thread.cpu_context.x20 = stk_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) p->thread.cpu_context.sp = (unsigned long)childregs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) ptrace_hw_copy_thread(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) void tls_preserve_current_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) *task_user_tls(current) = read_sysreg(tpidr_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) static void tls_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) tls_preserve_current_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (is_compat_thread(task_thread_info(next)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) write_sysreg(next->thread.uw.tp_value, tpidrro_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) else if (!arm64_kernel_unmapped_at_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) write_sysreg(0, tpidrro_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) write_sysreg(*task_user_tls(next), tpidr_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /* Restore the UAO state depending on next's addr_limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) void uao_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (IS_ENABLED(CONFIG_ARM64_UAO)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (task_thread_info(next)->addr_limit == KERNEL_DS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Force SSBS state on context-switch, since it may be lost after migrating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * from a CPU which treats the bit as RES0 in a heterogeneous system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static void ssbs_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * Nothing to do for kernel threads, but 'regs' may be junk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * (e.g. idle task) so check the flags and bail early.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (unlikely(next->flags & PF_KTHREAD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * If all CPUs implement the SSBS extension, then we just need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * context-switch the PSTATE field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (cpus_have_const_cap(ARM64_SSBS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) spectre_v4_enable_task_mitigation(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * We store our current task in sp_el0, which is clobbered by userspace. Keep a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * shadow copy so that we can restore this upon entry from userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * This is *only* for exception entry from EL0, and is not valid until we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * __switch_to() a user task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) DEFINE_PER_CPU(struct task_struct *, __entry_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static void entry_task_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) __this_cpu_write(__entry_task, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * Ensure access is disabled when switching to a 32bit task, ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * access is enabled when switching to a 64bit task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) static void erratum_1418040_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (is_compat_thread(task_thread_info(next)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static void erratum_1418040_new_exec(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) erratum_1418040_thread_switch(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * this function must be called with preemption disabled and the update to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * sctlr_user must be made in the same preemption disabled block so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * __switch_to() does not see the variable update before the SCTLR_EL1 one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) void update_sctlr_el1(u64 sctlr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * EnIA must not be cleared while in the kernel as this is necessary for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * in-kernel PAC. It will be cleared on kernel exit if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) /* ISB required for the kernel uaccess routines when setting TCF0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * Thread switching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) struct task_struct *last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) fpsimd_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) tls_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) hw_breakpoint_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) contextidr_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) entry_task_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) uao_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) ssbs_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) erratum_1418040_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) ptrauth_thread_switch_user(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * Complete any pending TLB or cache maintenance on this CPU in case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * the thread migrates to a different CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * This full barrier is also required by the membarrier system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) dsb(ish);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * MTE thread switching must happen after the DSB above to ensure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * any asynchronous tag check faults have been logged in the TFSR*_EL1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) mte_thread_switch(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /* avoid expensive SCTLR_EL1 accesses if no change */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (prev->thread.sctlr_user != next->thread.sctlr_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) update_sctlr_el1(next->thread.sctlr_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) trace_android_vh_is_fpsimd_save(prev, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) /* the actual thread switch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) last = cpu_switch_to(prev, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) unsigned long get_wchan(struct task_struct *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) struct stackframe frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) unsigned long stack_page, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (!p || p == current || p->state == TASK_RUNNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) stack_page = (unsigned long)try_get_task_stack(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (!stack_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) start_backtrace(&frame, thread_saved_fp(p), thread_saved_pc(p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (unwind_frame(p, &frame))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (!in_sched_functions(frame.pc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) ret = frame.pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) } while (count ++ < 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) put_task_stack(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) EXPORT_SYMBOL_GPL(get_wchan);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) unsigned long arch_align_stack(unsigned long sp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) sp -= get_random_int() & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return sp & ~0xf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) void arch_setup_new_exec(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) unsigned long mmflags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (is_compat_task()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) mmflags = MMCF_AARCH32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * Restrict the CPU affinity mask for a 32-bit task so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * it contains only 32-bit-capable CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * From the perspective of the task, this looks similar to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * what would happen if the 64-bit-only CPUs were hot-unplugged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * at the point of execve(), although we try a bit harder to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * honour the cpuset hierarchy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) force_compatible_cpus_allowed_ptr(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) current->mm->context.flags = mmflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) ptrauth_thread_init_user();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) mte_thread_init_user();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) erratum_1418040_new_exec();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (task_spec_ssb_noexec(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) PR_SPEC_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * Control the relaxed ABI allowing tagged user addresses into the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) static unsigned int tagged_addr_disabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) struct thread_info *ti = task_thread_info(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (is_compat_thread(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (system_supports_mte())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) valid_mask |= PR_MTE_TCF_MASK | PR_MTE_TAG_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (arg & ~valid_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * Do not allow the enabling of the tagged address ABI if globally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * disabled via sysctl abi.tagged_addr_disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (set_mte_ctrl(task, arg) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) long get_tagged_addr_ctrl(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) long ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct thread_info *ti = task_thread_info(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (is_compat_thread(ti))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) ret = PR_TAGGED_ADDR_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) ret |= get_mte_ctrl(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * Global sysctl to disable the tagged user addresses support. This control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * only prevents the tagged address ABI enabling via prctl() and does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * disable it for tasks that already opted in to the relaxed ABI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) static struct ctl_table tagged_addr_sysctl_table[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) .procname = "tagged_addr_disabled",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) .mode = 0644,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) .data = &tagged_addr_disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) .maxlen = sizeof(int),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) .proc_handler = proc_dointvec_minmax,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) .extra1 = SYSCTL_ZERO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) .extra2 = SYSCTL_ONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) static int __init tagged_addr_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (!register_sysctl("abi", tagged_addr_sysctl_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) core_initcall(tagged_addr_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) asmlinkage void __sched arm64_preempt_schedule_irq(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * Preempting a task from an IRQ means we leave copies of PSTATE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * on the stack. cpufeature's enable calls may modify PSTATE, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * resuming one of these preempted tasks would undo those changes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * Only allow a task to be preempted once cpufeatures have been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (system_capabilities_finalized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) preempt_schedule_irq();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) #ifdef CONFIG_BINFMT_ELF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) bool has_interp, bool is_interp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * For dynamically linked executables the interpreter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * responsible for setting PROT_BTI on everything except
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) if (is_interp != has_interp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (!(state->flags & ARM64_ELF_BTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (prot & PROT_EXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) prot |= PROT_BTI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) return prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags