^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * FP/SIMD context switching and fault handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2012 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author: Catalin Marinas <catalin.marinas@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/bitmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bottom_half.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/cpu_pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/irqflags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/prctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/sched/task_stack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/stddef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/swab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/esr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/exception.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/fpsimd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/cputype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/neon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/simd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/sigcontext.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/sysreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/virt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define FPEXC_IOF (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define FPEXC_DZF (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define FPEXC_OFF (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define FPEXC_UFF (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define FPEXC_IXF (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define FPEXC_IDF (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * In order to reduce the number of times the FPSIMD state is needlessly saved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * and restored, we need to keep track of two things:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * (a) for each task, we need to remember which CPU was the last one to have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * the task's FPSIMD state loaded into its FPSIMD registers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * (b) for each CPU, we need to remember which task's userland FPSIMD state has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * been loaded into its FPSIMD registers most recently, or whether it has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * been used to perform kernel mode NEON in the meantime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * the id of the current CPU every time the state is loaded onto a CPU. For (b),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * address of the userland FPSIMD state of the task that was loaded onto the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * the most recently, or NULL if kernel mode NEON has been performed after that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * With this in place, we no longer have to restore the next FPSIMD state right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * when switching between tasks. Instead, we can defer this check to userland
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * resume, at which time we verify whether the CPU's fpsimd_last_state and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * task's fpsimd_cpu are still mutually in sync. If this is the case, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * can omit the FPSIMD restore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * indicate whether or not the userland FPSIMD state of the current task is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * present in the registers. The flag is set unless the FPSIMD registers of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * CPU currently contain the most recent userland FPSIMD state of the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * save the task's FPSIMD context back to task_struct from softirq context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * To prevent this from racing with the manipulation of the task's FPSIMD state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * from task context and thereby corrupting the state, it is necessary to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * run but prevent them to use FPSIMD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * For a certain task, the sequence may look something like this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * - the task gets scheduled in; if both the task's fpsimd_cpu field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * cleared, otherwise it is set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * userland FPSIMD state is copied from memory to the registers, the task's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * fpsimd_cpu field is set to the id of the current CPU, the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * TIF_FOREIGN_FPSTATE flag is cleared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * - the task executes an ordinary syscall; upon return to userland, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * restored;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * - the task executes a syscall which executes some NEON instructions; this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * register contents to memory, clears the fpsimd_last_state per-cpu variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * and sets the TIF_FOREIGN_FPSTATE flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * - the task gets preempted after kernel_neon_end() is called; as we have not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * whatever is in the FPSIMD registers is not saved to memory, but discarded.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct fpsimd_last_state_struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct user_fpsimd_state *st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) void *sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) unsigned int sve_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Default VL for tasks that don't set it explicitly: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) static int __sve_default_vl = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static int get_sve_default_vl(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) return READ_ONCE(__sve_default_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #ifdef CONFIG_ARM64_SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static void set_sve_default_vl(int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) WRITE_ONCE(__sve_default_vl, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* Maximum supported vector length across all CPUs (initially poisoned) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) int __ro_after_init sve_max_vl = SVE_VL_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * Set of available vector lengths,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * where length vq encoded as bit __vq_to_bit(vq):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /* Set of vector lengths present on at least one cpu: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static void __percpu *efi_sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #else /* ! CONFIG_ARM64_SVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* Dummy declaration for code that will be optimised out: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) extern void __percpu *efi_sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #endif /* ! CONFIG_ARM64_SVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) DEFINE_PER_CPU(bool, fpsimd_context_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static void __get_cpu_fpsimd_context(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) WARN_ON(busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * Claim ownership of the CPU FPSIMD context for use by the calling context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * The caller may freely manipulate the FPSIMD context metadata until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * put_cpu_fpsimd_context() is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * The double-underscore version must only be called if you know the task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * can't be preempted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) static void get_cpu_fpsimd_context(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) __get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static void __put_cpu_fpsimd_context(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * Release the CPU FPSIMD context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * Must be called from a context in which get_cpu_fpsimd_context() was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * previously called, with no call to put_cpu_fpsimd_context() in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * meantime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static void put_cpu_fpsimd_context(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) __put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static bool have_cpu_fpsimd_context(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return !preemptible() && __this_cpu_read(fpsimd_context_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Call __sve_free() directly only if you know task can't be scheduled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * or preempted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static void __sve_free(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) kfree(task->thread.sve_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) task->thread.sve_state = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static void sve_free(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) __sve_free(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * TIF_SVE controls whether a task can use SVE without trapping while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * in userspace, and also the way a task's FPSIMD/SVE state is stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * in thread_struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * The kernel uses this flag to track whether a user task is actively
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * using SVE, and therefore whether full SVE register state needs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * be tracked. If not, the cheaper FPSIMD context handling code can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * be used instead of the more costly SVE equivalents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * * TIF_SVE set:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * The task can execute SVE instructions while in userspace without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * trapping to the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * corresponding Zn), P0-P15 and FFR are encoded in in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * task->thread.sve_state, formatted appropriately for vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * length task->thread.sve_vl.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * task->thread.sve_state must point to a valid buffer at least
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * sve_state_size(task) bytes in size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * During any syscall, the kernel may optionally clear TIF_SVE and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * discard the vector state except for the FPSIMD subset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * * TIF_SVE clear:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * An attempt by the user task to execute an SVE instruction causes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * do_sve_acc() to be called, which does some preparation and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * sets TIF_SVE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * When stored, FPSIMD registers V0-V31 are encoded in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * logically zero but not stored anywhere; P0-P15 and FFR are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * stored and have unspecified values from userspace's point of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * view. For hygiene purposes, the kernel zeroes them on next use,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * but userspace is discouraged from relying on this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * task->thread.sve_state does not need to be non-NULL, valid or any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * particular size: it must not be dereferenced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * irrespective of whether TIF_SVE is clear or set, since these are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * not vector length dependent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * Update current's FPSIMD/SVE registers from thread_struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * This function should be called only when the FPSIMD/SVE state in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * thread_struct is known to be up to date, when preparing to enter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static void task_fpsimd_load(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) WARN_ON(!system_supports_fpsimd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) WARN_ON(!have_cpu_fpsimd_context());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (system_supports_sve() && test_thread_flag(TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) sve_load_state(sve_pffr(¤t->thread),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ¤t->thread.uw.fpsimd_state.fpsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) sve_vq_from_vl(current->thread.sve_vl) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) fpsimd_load_state(¤t->thread.uw.fpsimd_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * date with respect to the CPU registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static void fpsimd_save(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct fpsimd_last_state_struct const *last =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) this_cpu_ptr(&fpsimd_last_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) WARN_ON(!system_supports_fpsimd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) WARN_ON(!have_cpu_fpsimd_context());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (WARN_ON(sve_get_vl() != last->sve_vl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Can't save the user regs, so current would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * re-enter user with corrupt state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * There's no way to recover, so kill it:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) sve_save_state((char *)last->sve_state +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) sve_ffr_offset(last->sve_vl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) &last->st->fpsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) fpsimd_save_state(last->st);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * All vector length selection from userspace comes through here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * We're on a slow path, so some sanity-checks are included.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * If things go wrong there's a bug somewhere, but try to fall back to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * safe choice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static unsigned int find_supported_vector_length(unsigned int vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) int max_vl = sve_max_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (WARN_ON(!sve_vl_valid(vl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) vl = SVE_VL_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (WARN_ON(!sve_vl_valid(max_vl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) max_vl = SVE_VL_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) if (vl > max_vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) vl = max_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) __vq_to_bit(sve_vq_from_vl(vl)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return sve_vl_from_vq(__bit_to_vq(bit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int sve_proc_do_default_vl(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) int vl = get_sve_default_vl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct ctl_table tmp_table = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) .data = &vl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) .maxlen = sizeof(vl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (ret || !write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /* Writing -1 has the special meaning "set to max": */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (vl == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) vl = sve_max_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (!sve_vl_valid(vl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) set_sve_default_vl(find_supported_vector_length(vl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static struct ctl_table sve_default_vl_table[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) .procname = "sve_default_vector_length",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) .mode = 0644,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) .proc_handler = sve_proc_do_default_vl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static int __init sve_sysctl_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (!register_sysctl("abi", sve_default_vl_table))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) #else /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static int __init sve_sysctl_init(void) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) #endif /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) #ifdef CONFIG_CPU_BIG_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static __uint128_t arm64_cpu_to_le128(__uint128_t x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) u64 a = swab64(x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) u64 b = swab64(x >> 64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) return ((__uint128_t)a << 64) | b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) static __uint128_t arm64_cpu_to_le128(__uint128_t x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) unsigned int vq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) __uint128_t *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) for (i = 0; i < SVE_NUM_ZREGS; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) p = (__uint128_t *)ZREG(sst, vq, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) *p = arm64_cpu_to_le128(fst->vregs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * task->thread.sve_state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * Task can be a non-runnable task, or current. In the latter case,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * the caller must have ownership of the cpu FPSIMD context before calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * task->thread.sve_state must point to at least sve_state_size(task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * bytes of allocated kernel memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * task->thread.uw.fpsimd_state must be up to date before calling this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) static void fpsimd_to_sve(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) unsigned int vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) void *sst = task->thread.sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (!system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) vq = sve_vq_from_vl(task->thread.sve_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) __fpsimd_to_sve(sst, fst, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * Transfer the SVE state in task->thread.sve_state to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * task->thread.uw.fpsimd_state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * Task can be a non-runnable task, or current. In the latter case,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * the caller must have ownership of the cpu FPSIMD context before calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * task->thread.sve_state must point to at least sve_state_size(task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * bytes of allocated kernel memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * task->thread.sve_state must be up to date before calling this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static void sve_to_fpsimd(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) unsigned int vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) void const *sst = task->thread.sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) __uint128_t const *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (!system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) vq = sve_vq_from_vl(task->thread.sve_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) for (i = 0; i < SVE_NUM_ZREGS; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) p = (__uint128_t const *)ZREG(sst, vq, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) fst->vregs[i] = arm64_le128_to_cpu(*p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) #ifdef CONFIG_ARM64_SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * Return how many bytes of memory are required to store the full SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * state for task, given task's currently configured vector length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) size_t sve_state_size(struct task_struct const *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * Ensure that task->thread.sve_state is allocated and sufficiently large.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * This function should be used only in preparation for replacing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * task->thread.sve_state with new data. The memory is always zeroed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * here to prevent stale data from showing through: this is done in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * the interest of testability and predictability: except in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * do_sve_acc() case, there is no ABI requirement to hide stale data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * written previously be task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) void sve_alloc(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (task->thread.sve_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) memset(task->thread.sve_state, 0, sve_state_size(task));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* This is a small allocation (maximum ~8KB) and Should Not Fail. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) task->thread.sve_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) kzalloc(sve_state_size(task), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * If future SVE revisions can have larger vectors though,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * this may cease to be true:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) BUG_ON(!task->thread.sve_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * Ensure that task->thread.sve_state is up to date with respect to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * the user task, irrespective of when SVE is in use or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * This should only be called by ptrace. task must be non-runnable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * task->thread.sve_state must point to at least sve_state_size(task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * bytes of allocated kernel memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) void fpsimd_sync_to_sve(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (!test_tsk_thread_flag(task, TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) fpsimd_to_sve(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * Ensure that task->thread.uw.fpsimd_state is up to date with respect to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * the user task, irrespective of whether SVE is in use or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * This should only be called by ptrace. task must be non-runnable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * task->thread.sve_state must point to at least sve_state_size(task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * bytes of allocated kernel memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) void sve_sync_to_fpsimd(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) if (test_tsk_thread_flag(task, TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) sve_to_fpsimd(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * Ensure that task->thread.sve_state is up to date with respect to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * the task->thread.uw.fpsimd_state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * This should only be called by ptrace to merge new FPSIMD register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * values into a task for which SVE is currently active.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * task must be non-runnable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * task->thread.sve_state must point to at least sve_state_size(task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * bytes of allocated kernel memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * task->thread.uw.fpsimd_state must already have been initialised with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * the new FPSIMD register values to be merged in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) unsigned int vq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) void *sst = task->thread.sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (!test_tsk_thread_flag(task, TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) vq = sve_vq_from_vl(task->thread.sve_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) __fpsimd_to_sve(sst, fst, vq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) int sve_set_vector_length(struct task_struct *task,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) unsigned long vl, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) PR_SVE_SET_VL_ONEXEC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (!sve_vl_valid(vl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * Clamp to the maximum vector length that VL-agnostic SVE code can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * work with. A flag may be assigned in the future to allow setting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * of larger vector lengths without confusing older software.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (vl > SVE_VL_ARCH_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) vl = SVE_VL_ARCH_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) vl = find_supported_vector_length(vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (flags & (PR_SVE_VL_INHERIT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) PR_SVE_SET_VL_ONEXEC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) task->thread.sve_vl_onexec = vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /* Reset VL to system default on next exec: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) task->thread.sve_vl_onexec = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* Only actually set the VL if not deferred: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (flags & PR_SVE_SET_VL_ONEXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (vl == task->thread.sve_vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * To ensure the FPSIMD bits of the SVE vector registers are preserved,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * write any live register state back to task_struct, and convert to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * non-SVE thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (task == current) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) fpsimd_flush_task_state(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) sve_to_fpsimd(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (task == current)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * Force reallocation of task SVE state to the correct size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * on next use:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) sve_free(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) task->thread.sve_vl = vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) flags & PR_SVE_VL_INHERIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * Encode the current vector length and flags for return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * This is only required for prctl(): ptrace has separate fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * flags are as for sve_set_vector_length().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) static int sve_prctl_status(unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (flags & PR_SVE_SET_VL_ONEXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) ret = current->thread.sve_vl_onexec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) ret = current->thread.sve_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (test_thread_flag(TIF_SVE_VL_INHERIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) ret |= PR_SVE_VL_INHERIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /* PR_SVE_SET_VL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) int sve_set_current_vl(unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) unsigned long vl, flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) vl = arg & PR_SVE_VL_LEN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) flags = arg & ~vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (!system_supports_sve() || is_compat_task())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) ret = sve_set_vector_length(current, vl, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return sve_prctl_status(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) /* PR_SVE_GET_VL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) int sve_get_current_vl(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) if (!system_supports_sve() || is_compat_task())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return sve_prctl_status(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) unsigned int vq, vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) unsigned long zcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) bitmap_zero(map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) zcr = ZCR_ELx_LEN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) vl = sve_get_vl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) vq = sve_vq_from_vl(vl); /* skip intervening lengths */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) set_bit(__vq_to_bit(vq), map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * Initialise the set of known supported VQs for the boot CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) * This is called during kernel boot, before secondary CPUs are brought up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) void __init sve_init_vq_map(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) sve_probe_vqs(sve_vq_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * If we haven't committed to the set of supported VQs yet, filter out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * those not supported by the current CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * This function is called during the bring-up of early secondary CPUs only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) void sve_update_vq_map(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) sve_probe_vqs(tmp_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * Check whether the current CPU supports all VQs in the committed set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * This function is called during the bring-up of late secondary CPUs only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) int sve_verify_vq_map(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) unsigned long b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) sve_probe_vqs(tmp_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * For KVM, it is necessary to ensure that this CPU doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * support any vector length that guests may have probed as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * unsupported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /* Recover the set of supported VQs: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) /* Find VQs supported that are not globally supported: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /* Find the lowest such VQ, if any: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) b = find_last_bit(tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (b >= SVE_VQ_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) return 0; /* no mismatches */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * Mismatches above sve_max_virtualisable_vl are fine, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) static void __init sve_efi_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (!IS_ENABLED(CONFIG_EFI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * alloc_percpu() warns and prints a backtrace if this goes wrong.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * This is evidence of a crippled system and we are returning void,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * so no attempt is made to handle this situation here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (!sve_vl_valid(sve_max_vl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) efi_sve_state = __alloc_percpu(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (!efi_sve_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) panic("Cannot allocate percpu memory for EFI SVE save/restore");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * Enable SVE for EL1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * Intended for use by the cpufeatures code during CPU boot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * vector length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * Use only if SVE is present.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * This function clobbers the SVE vector length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) u64 read_zcr_features(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) u64 zcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) unsigned int vq_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * Set the maximum possible VL, and write zeroes to all other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * bits to see if they stick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) sve_kernel_enable(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) zcr = read_sysreg_s(SYS_ZCR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) vq_max = sve_vq_from_vl(sve_get_vl());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) zcr |= vq_max - 1; /* set LEN field to maximum effective value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return zcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) void __init sve_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) u64 zcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) unsigned long b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) if (!system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * The SVE architecture mandates support for 128-bit vectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * so sve_vq_map must have at least SVE_VQ_MIN set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * If something went wrong, at least try to patch it up:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * Sanity-check that the max VL we determined through CPU features
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * corresponds properly to sve_vq_map. If not, do our best:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) sve_max_vl = find_supported_vector_length(sve_max_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * For the default VL, pick the maximum supported value <= 64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * VL == 64 is guaranteed not to grow the signal frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) set_sve_default_vl(find_supported_vector_length(64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) b = find_last_bit(tmp_map, SVE_VQ_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (b >= SVE_VQ_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /* No non-virtualisable VLs found */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) sve_max_virtualisable_vl = SVE_VQ_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) else if (WARN_ON(b == SVE_VQ_MAX - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /* No virtualisable VLs? This is architecturally forbidden. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) sve_max_virtualisable_vl = SVE_VQ_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) else /* b + 1 < SVE_VQ_MAX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) if (sve_max_virtualisable_vl > sve_max_vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) sve_max_virtualisable_vl = sve_max_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) pr_info("SVE: maximum available vector length %u bytes per vector\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) sve_max_vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) pr_info("SVE: default vector length %u bytes per vector\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) get_sve_default_vl());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) /* KVM decides whether to support mismatched systems. Just warn here: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (sve_max_virtualisable_vl < sve_max_vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) pr_warn("SVE: unvirtualisable vector lengths present\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) sve_efi_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * Called from the put_task_struct() path, which cannot get here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * unless dead_task is really dead and not schedulable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) void fpsimd_release_task(struct task_struct *dead_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) __sve_free(dead_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) #endif /* CONFIG_ARM64_SVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * Trapped SVE access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * Storage is allocated for the full SVE state, the current FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * register contents are migrated across, and TIF_SVE is set so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * the SVE access trap will be disabled the next time this task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * reaches ret_to_user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * would have disabled the SVE access trap for userspace during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * ret_to_user, making an SVE access trap impossible in that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) void do_sve_acc(unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /* Even if we chose not to use SVE, the hardware could still trap: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) sve_alloc(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) /* Force ret_to_user to reload the registers: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) fpsimd_flush_task_state(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) fpsimd_to_sve(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) if (test_and_set_thread_flag(TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) WARN_ON(1); /* SVE access shouldn't have trapped */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * Trapped FP/ASIMD access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /* TODO: implement lazy context saving/restoring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * Raise a SIGFPE for the current process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) unsigned int si_code = FPE_FLTUNK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) if (esr & ESR_ELx_FP_EXC_TFV) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) if (esr & FPEXC_IOF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) si_code = FPE_FLTINV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) else if (esr & FPEXC_DZF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) si_code = FPE_FLTDIV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) else if (esr & FPEXC_OFF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) si_code = FPE_FLTOVF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) else if (esr & FPEXC_UFF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) si_code = FPE_FLTUND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) else if (esr & FPEXC_IXF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) si_code = FPE_FLTRES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) send_sig_fault(SIGFPE, si_code,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) (void __user *)instruction_pointer(regs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) void fpsimd_thread_switch(struct task_struct *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) bool wrong_task, wrong_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) __get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) /* Save unsaved fpsimd state, if any: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) * state. For kernel threads, FPSIMD registers are never loaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * and wrong_task and wrong_cpu will always be true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) &next->thread.uw.fpsimd_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) wrong_task || wrong_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) __put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) void fpsimd_flush_thread(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) int vl, supported_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) fpsimd_flush_task_state(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) memset(¤t->thread.uw.fpsimd_state, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) sizeof(current->thread.uw.fpsimd_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) if (system_supports_sve()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) clear_thread_flag(TIF_SVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) sve_free(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * Reset the task vector length as required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * This is where we ensure that all user tasks have a valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * vector length configured: no kernel task can become a user
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * task without an exec and hence a call to this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * By the time the first call to this function is made, all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * early hardware probing is complete, so __sve_default_vl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * should be valid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) * If a bug causes this to go wrong, we make some noise and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * try to fudge thread.sve_vl to a safe value here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) vl = current->thread.sve_vl_onexec ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) current->thread.sve_vl_onexec : get_sve_default_vl();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) if (WARN_ON(!sve_vl_valid(vl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) vl = SVE_VL_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) supported_vl = find_supported_vector_length(vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (WARN_ON(supported_vl != vl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) vl = supported_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) current->thread.sve_vl = vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) * If the task is not set to inherit, ensure that the vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * length will be reset by a subsequent exec:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (!test_thread_flag(TIF_SVE_VL_INHERIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) current->thread.sve_vl_onexec = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * Save the userland FPSIMD state of 'current' to memory, but only if the state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) * currently held in the registers does in fact belong to 'current'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) void fpsimd_preserve_current_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * Like fpsimd_preserve_current_state(), but ensure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * current->thread.uw.fpsimd_state is updated so that it can be copied to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * the signal frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) void fpsimd_signal_preserve_current_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) fpsimd_preserve_current_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) if (system_supports_sve() && test_thread_flag(TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) sve_to_fpsimd(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * Associate current's FPSIMD context with this cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * The caller must have ownership of the cpu FPSIMD context before calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) void fpsimd_bind_task_to_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) struct fpsimd_last_state_struct *last =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) this_cpu_ptr(&fpsimd_last_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) WARN_ON(!system_supports_fpsimd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) last->st = ¤t->thread.uw.fpsimd_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) last->sve_state = current->thread.sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) last->sve_vl = current->thread.sve_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) current->thread.fpsimd_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (system_supports_sve()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) /* Toggle SVE trapping for userspace if needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) if (test_thread_flag(TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) sve_user_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) sve_user_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /* Serialised by exception return to user */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) unsigned int sve_vl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) struct fpsimd_last_state_struct *last =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) this_cpu_ptr(&fpsimd_last_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) WARN_ON(!system_supports_fpsimd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) WARN_ON(!in_softirq() && !irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) last->st = st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) last->sve_state = sve_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) last->sve_vl = sve_vl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * Load the userland FPSIMD state of 'current' from memory, but only if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) * state of 'current'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) void fpsimd_restore_current_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * For the tasks that were created before we detected the absence of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * FP/SIMD, the TIF_FOREIGN_FPSTATE could be set via fpsimd_thread_switch(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) * e.g, init. This could be then inherited by the children processes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * If we later detect that the system doesn't support FP/SIMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * we must clear the flag for all the tasks to indicate that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) * FPSTATE is clean (as we can't have one) to avoid looping for ever in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) * do_notify_resume().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) if (!system_supports_fpsimd()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) clear_thread_flag(TIF_FOREIGN_FPSTATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) task_fpsimd_load();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) fpsimd_bind_task_to_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * Load an updated userland FPSIMD state for 'current' from memory and set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) * flag that indicates that the FPSIMD register contents are the most recent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) * FPSIMD state of 'current'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) void fpsimd_update_current_state(struct user_fpsimd_state const *state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (WARN_ON(!system_supports_fpsimd()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) current->thread.uw.fpsimd_state = *state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (system_supports_sve() && test_thread_flag(TIF_SVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) fpsimd_to_sve(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) task_fpsimd_load();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) fpsimd_bind_task_to_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) clear_thread_flag(TIF_FOREIGN_FPSTATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) * Invalidate live CPU copies of task t's FPSIMD state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * This function may be called with preemption enabled. The barrier()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * ensures that the assignment to fpsimd_cpu is visible to any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * preemption/softirq that could race with set_tsk_thread_flag(), so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * subsequent code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) void fpsimd_flush_task_state(struct task_struct *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) t->thread.fpsimd_cpu = NR_CPUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * If we don't support fpsimd, bail out after we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * reset the fpsimd_cpu for this task and clear the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * FPSTATE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * Invalidate any task's FPSIMD state that is present on this cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * before calling this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) static void fpsimd_flush_cpu_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) WARN_ON(!system_supports_fpsimd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) __this_cpu_write(fpsimd_last_state.st, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) set_thread_flag(TIF_FOREIGN_FPSTATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * Save the FPSIMD state to memory and invalidate cpu view.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * This function must be called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) void fpsimd_save_and_flush_cpu_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) WARN_ON(preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) __get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) fpsimd_flush_cpu_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) __put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) #ifdef CONFIG_KERNEL_MODE_NEON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * Kernel-side NEON support functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) * Must not be called unless may_use_simd() returns true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * Task context in the FPSIMD registers is saved back to memory as necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * A matching call to kernel_neon_end() must be made before returning from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * calling context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * The caller may freely use the FPSIMD registers until kernel_neon_end() is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) void kernel_neon_begin(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (WARN_ON(!system_supports_fpsimd()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) BUG_ON(!may_use_simd());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) get_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) /* Save unsaved fpsimd state, if any: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) fpsimd_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /* Invalidate any task state remaining in the fpsimd regs: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) fpsimd_flush_cpu_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) EXPORT_SYMBOL(kernel_neon_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) * Must be called from a context in which kernel_neon_begin() was previously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) * called, with no call to kernel_neon_end() in the meantime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) * The caller must not use the FPSIMD registers after this function is called,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) * unless kernel_neon_begin() is called again in the meantime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) void kernel_neon_end(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) put_cpu_fpsimd_context();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) EXPORT_SYMBOL(kernel_neon_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) #ifdef CONFIG_EFI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) static DEFINE_PER_CPU(bool, efi_sve_state_used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * EFI runtime services support functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) * This means that for EFI (and only for EFI), we have to assume that FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) * is always used rather than being an optional accelerator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) * These functions provide the necessary support for ensuring FPSIMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) * save/restore in the contexts from which EFI is used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) * Do not use them for any other purpose -- if tempted to do so, you are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) * either doing something wrong or you need to propose some refactoring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) void __efi_fpsimd_begin(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) WARN_ON(preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (may_use_simd()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) kernel_neon_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) * If !efi_sve_state, SVE can't be in use yet and doesn't need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * preserving:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (system_supports_sve() && likely(efi_sve_state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) char *sve_state = this_cpu_ptr(efi_sve_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) __this_cpu_write(efi_sve_state_used, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) __this_cpu_write(efi_fpsimd_state_used, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) void __efi_fpsimd_end(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (!system_supports_fpsimd())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) kernel_neon_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (system_supports_sve() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) likely(__this_cpu_read(efi_sve_state_used))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) char const *sve_state = this_cpu_ptr(efi_sve_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) sve_vq_from_vl(sve_get_vl()) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) __this_cpu_write(efi_sve_state_used, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) #endif /* CONFIG_EFI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) #endif /* CONFIG_KERNEL_MODE_NEON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) #ifdef CONFIG_CPU_PM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) unsigned long cmd, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) case CPU_PM_ENTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) fpsimd_save_and_flush_cpu_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) case CPU_PM_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) case CPU_PM_ENTER_FAILED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) static struct notifier_block fpsimd_cpu_pm_notifier_block = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) .notifier_call = fpsimd_cpu_pm_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) static void __init fpsimd_pm_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) static inline void fpsimd_pm_init(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) #endif /* CONFIG_CPU_PM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) static int fpsimd_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) per_cpu(fpsimd_last_state.st, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) static inline void fpsimd_hotplug_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) NULL, fpsimd_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) static inline void fpsimd_hotplug_init(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * FP/SIMD support code initialisation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) static int __init fpsimd_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (cpu_have_named_feature(FP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) fpsimd_pm_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) fpsimd_hotplug_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) pr_notice("Floating-point is not implemented\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (!cpu_have_named_feature(ASIMD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) pr_notice("Advanced SIMD is not implemented\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) return sve_sysctl_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) core_initcall(fpsimd_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags