^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Contains CPU feature definitions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2015 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * A note for the weary kernel hacker: the code here is confusing and hard to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * follow! That's partly because it's solving a nasty problem, but also because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * there's a little bit of over-abstraction that tends to obscure what's going
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * on behind a maze of helper functions and macros.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * The basic problem is that hardware folks have started gluing together CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * with distinct architectural features; in some cases even creating SoCs where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * user-visible instructions are available only on a subset of the available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * cores. We try to address this by snapshotting the feature registers of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * boot CPU and comparing these with the feature registers of each secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * CPU when bringing them up. If there is a mismatch, then we update the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * snapshot state to indicate the lowest-common denominator of the feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * known as the "safe" value. This snapshot state can be queried to view the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * "sanitised" value of a feature register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * The sanitised register values are used to decide which capabilities we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * have in the system. These may be in the form of traditional "hwcaps"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * advertised to userspace or internal "cpucaps" which are used to configure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * things like alternative patching and static keys. While a feature mismatch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * may result in a TAINT_CPU_OUT_OF_SPEC kernel taint, a capability mismatch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * may prevent a CPU from being onlined at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * Some implementation details worth remembering:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * - Mismatched features are *always* sanitised to a "safe" value, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * usually indicates that the feature is not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * - A mismatched feature marked with FTR_STRICT will cause a "SANITY CHECK"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * warning when onlining an offending CPU and the kernel will be tainted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * with TAINT_CPU_OUT_OF_SPEC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * - Features marked as FTR_VISIBLE have their sanitised value visible to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * userspace. FTR_VISIBLE features in registers that are only visible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * to EL0 by trapping *must* have a corresponding HWCAP so that late
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * onlining of CPUs cannot lead to features disappearing at runtime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * - A "feature" is typically a 4-bit register field. A "capability" is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * high-level description derived from the sanitised field value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * - Read the Arm ARM (DDI 0487F.a) section D13.1.3 ("Principles of the ID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * scheme for fields in ID registers") to understand when feature fields
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * may be signed or unsigned (FTR_SIGNED and FTR_UNSIGNED accordingly).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * - KVM exposes its own view of the feature registers to guest operating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * systems regardless of FTR_VISIBLE. This is typically driven from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * sanitised register values to allow virtual CPUs to be migrated between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * arbitrary physical CPUs, but some features not present on the host are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * also advertised and emulated. Look at sys_reg_descs[] for the gory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * - If the arm64_ftr_bits[] for a register has a missing field, then this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * field is treated as STRICT RES0, including for read_sanitised_ftr_reg().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * This is stronger than FTR_HIDDEN and can be used to hide features from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * KVM guests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define pr_fmt(fmt) "CPU features: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <linux/bsearch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <linux/crash_dump.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #include <linux/sysfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #include <asm/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #include <asm/cpu_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #include <asm/fpsimd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #include <asm/kvm_host.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #include <asm/mte.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #include <asm/sysreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #include <asm/vectors.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #include <asm/virt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* Kernel representation of AT_HWCAP and AT_HWCAP2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static unsigned long elf_hwcap __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define COMPAT_ELF_HWCAP_DEFAULT \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) COMPAT_HWCAP_TLS|COMPAT_HWCAP_IDIV|\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) COMPAT_HWCAP_LPAE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned int compat_elf_hwcap2 __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) EXPORT_SYMBOL(cpu_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Need also bit for ARM64_CB_PATCH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) bool arm64_use_ng_mappings = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) EXPORT_SYMBOL(arm64_use_ng_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * support it?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static bool __read_mostly allow_mismatched_32bit_el0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * seen at least one CPU capable of 32-bit EL0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * Mask of CPUs supporting 32-bit EL0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * Only valid if arm64_mismatched_32bit_el0 is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * Flag to indicate if we have computed the system wide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * capabilities based on the boot time active CPUs. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * will be used to determine if a new booting CPU should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * go through the verification process to make sure that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * supports the system capabilities, without using a hotplug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * notifier. This is also used to decide if we could use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * the fast path for checking constant CPU caps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) EXPORT_SYMBOL(arm64_const_caps_ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) static inline void finalize_system_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static_branch_enable(&arm64_const_caps_ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) void dump_cpu_features(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* file-wide pr_fmt adds "CPU features: " prefix */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) EXPORT_SYMBOL(cpu_hwcap_keys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .sign = SIGNED, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) .visible = VISIBLE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) .strict = STRICT, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) .type = TYPE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) .shift = SHIFT, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) .width = WIDTH, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) .safe_val = SAFE_VAL, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /* Define a feature with unsigned values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* Define a feature with a signed value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define ARM64_FTR_END \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) .width = 0, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* meta feature for alternatives */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static bool __maybe_unused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static bool __system_matches_cap(unsigned int n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * NOTE: Any changes to the visibility of features should be kept in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * sync with the documentation of the CPU feature register ABI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RNDR_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TLB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_I8MM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DGH_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_BF16_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SPECRES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FRINTTS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPI_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_API_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_APA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_AMU_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_MPAM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SEL2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MPAMFRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_RASFRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_BT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_F64MM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_F32MM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_I8MM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SM4_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SHA3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_BF16_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_BITPERM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_AES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_SVEVER_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * Page size not being supported at Stage-2 is not fatal. You
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * just give up KVM if PAGE_SIZE isn't supported there. Go fix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * your favourite nesting hypervisor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * There is a small corner case where the hypervisor explicitly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * advertises a given granule size at Stage-2 (value 2) on some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * vCPUs, and uses the fallback to Stage-1 (value 0) for other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * vCPUs. Although this is not forbidden by the architecture, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * indicates that the hypervisor is being silly (or buggy).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * We make no effort to cope with this and pretend that if these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * fields are inconsistent across vCPUs, then it isn't worth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * trying to bring KVM up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_2_SHIFT, 4, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_2_SHIFT, 4, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_2_SHIFT, 4, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * We already refuse to boot CPUs that don't support our configured
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * page size, so we can only detect mismatches for a page size other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * than the one we're currently using. Unfortunately, SoCs like this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * exist in the wild so, even though we don't like it, we'll have to go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * along with it and treat them as non-strict.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /* Linux shouldn't care about secure memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * Differing PARange is fine as long as all peripherals and memory are mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * within the minimum PARange of all CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_AFP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_ETS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TWED_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_XNX_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_SPECSEI_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_E0PD_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EVT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_BBM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_TTL_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IDS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_ST_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_NV_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CCIDX_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) static const struct arm64_ftr_bits ftr_ctr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DIC_SHIFT, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IDC_SHIFT, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_CWG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_ERG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DMINLINE_SHIFT, 4, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * Linux can handle differing I-cache policies. Userspace JITs will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * make use of *minLine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * If we have differing I-cache policies, report it as the weakest - VIPT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, CTR_L1IP_SHIFT, 2, ICACHE_POLICY_VIPT), /* L1Ip */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IMINLINE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static struct arm64_ftr_override __ro_after_init no_override = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) .name = "SYS_CTR_EL0",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) .ftr_bits = ftr_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) .override = &no_override,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_INNERSHR_SHIFT, 4, 0xf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_FCSE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_AUXREG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_TCM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_SHARELVL_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_OUTERSHR_SHIFT, 4, 0xf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_PMSA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_VMSA_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_DOUBLELOCK_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * We can instantiate multiple PMU instances with different levels
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * of support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static const struct arm64_ftr_bits ftr_mvfr2[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static const struct arm64_ftr_bits ftr_dczid[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, DCZID_DZP_SHIFT, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, DCZID_BS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) static const struct arm64_ftr_bits ftr_id_isar0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_COPROC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_CMPBRANCH_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITFIELD_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_BITCOUNT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_SWAP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static const struct arm64_ftr_bits ftr_id_isar5[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EVT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CCIDX_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_LSM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_HPDS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_CNP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_XNX_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_AC2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * SpecSEI = 1 indicates that the PE might generate an SError on an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * external abort on speculative read. It is safe to assume that an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * SError might be generated than it will not be. Hence it has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * classified as FTR_HIGHER_SAFE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_SPECSEI_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static const struct arm64_ftr_bits ftr_id_isar4[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SWP_FRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_PSR_M_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SYNCH_PRIM_FRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_BARRIER_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_SMC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WRITEBACK_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_WITHSHIFTS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_UNPRIV_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_ETS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) static const struct arm64_ftr_bits ftr_id_isar6[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_I8MM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_BF16_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SPECRES_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_SB_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_FHM_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_DP_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_JSCVT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static const struct arm64_ftr_bits ftr_id_pfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_DIT_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_CSV2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE2_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE1_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_STATE0_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static const struct arm64_ftr_bits ftr_id_pfr1[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GIC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRT_FRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SEC_FRAC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_GENTIMER_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_VIRTUALIZATION_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_MPROGMOD_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_SECURITY_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_PROGMOD_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) static const struct arm64_ftr_bits ftr_id_pfr2[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static const struct arm64_ftr_bits ftr_id_dfr0[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) /* [31:28] TraceFilt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_PERFMON_SHIFT, 4, 0xf),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MPROFDBG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPTRC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPTRC_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_MMAPDBG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPSDBG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_COPDBG_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static const struct arm64_ftr_bits ftr_id_dfr1[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_MTPMU_SHIFT, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static const struct arm64_ftr_bits ftr_zcr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0), /* LEN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * Common ftr bits for a 32bit register with all hidden, strict
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * attributes, with 4bit feature fields and a default safe value of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * 0. Covers the following 32bit registers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * id_isar[1-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) static const struct arm64_ftr_bits ftr_generic_32bits[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /* Table for a single 32bit feature value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static const struct arm64_ftr_bits ftr_single32[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) static const struct arm64_ftr_bits ftr_raz[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) ARM64_FTR_END,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) #define __ARM64_FTR_REG_OVERRIDE(id_str, id, table, ovr) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) .sys_id = id, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) .reg = &(struct arm64_ftr_reg){ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) .name = id_str, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) .override = (ovr), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .ftr_bits = &((table)[0]), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) #define ARM64_FTR_REG_OVERRIDE(id, table, ovr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) __ARM64_FTR_REG_OVERRIDE(#id, id, table, ovr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) #define ARM64_FTR_REG(id, table) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) __ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) struct arm64_ftr_override __ro_after_init id_aa64mmfr1_override;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) struct arm64_ftr_override __ro_after_init id_aa64pfr1_override;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) struct arm64_ftr_override __ro_after_init id_aa64isar1_override;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static const struct __ftr_reg_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) u32 sys_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) struct arm64_ftr_reg *reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) } arm64_ftr_regs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) /* Op1 = 0, CRn = 0, CRm = 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_id_pfr1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) /* Op1 = 0, CRn = 0, CRm = 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_id_isar0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_id_isar4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) ARM64_FTR_REG(SYS_ID_ISAR6_EL1, ftr_id_isar6),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /* Op1 = 0, CRn = 0, CRm = 3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) ARM64_FTR_REG(SYS_ID_PFR2_EL1, ftr_id_pfr2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ARM64_FTR_REG(SYS_ID_DFR1_EL1, ftr_id_dfr1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) ARM64_FTR_REG(SYS_ID_MMFR5_EL1, ftr_id_mmfr5),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /* Op1 = 0, CRn = 0, CRm = 4 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) &id_aa64pfr1_override),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /* Op1 = 0, CRn = 0, CRm = 5 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /* Op1 = 0, CRn = 0, CRm = 6 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) &id_aa64isar1_override),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) ARM64_FTR_REG(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* Op1 = 0, CRn = 0, CRm = 7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) &id_aa64mmfr1_override),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /* Op1 = 0, CRn = 1, CRm = 2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /* Op1 = 3, CRn = 0, CRm = 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* Op1 = 3, CRn = 14, CRm = 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static int search_cmp_ftr_reg(const void *id, const void *regp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * get_arm64_ftr_reg_nowarn - Looks up a feature register entry using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * its sys_reg() encoding. With the array arm64_ftr_regs sorted in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) * ascending order of sys_id, we use binary search to find a matching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * returns - Upon success, matching ftr_reg entry for id.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * - NULL on failure. It is upto the caller to decide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * the impact of a failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) const struct __ftr_reg_entry *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) ret = bsearch((const void *)(unsigned long)sys_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) arm64_ftr_regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) ARRAY_SIZE(arm64_ftr_regs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) sizeof(arm64_ftr_regs[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) search_cmp_ftr_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) return ret->reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * get_arm64_ftr_reg - Looks up a feature register entry using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * its sys_reg() encoding. This calls get_arm64_ftr_reg_nowarn().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * returns - Upon success, matching ftr_reg entry for id.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * - NULL on failure but with an WARN_ON().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct arm64_ftr_reg *reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) reg = get_arm64_ftr_reg_nowarn(sys_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * Requesting a non-existent register search is an error. Warn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * and let the caller handle it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) WARN_ON(!reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) s64 ftr_val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) u64 mask = arm64_ftr_mask(ftrp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) reg &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) reg |= (ftr_val << ftrp->shift) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) s64 cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) s64 ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) switch (ftrp->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) case FTR_EXACT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) ret = ftrp->safe_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) case FTR_LOWER_SAFE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) ret = new < cur ? new : cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) case FTR_HIGHER_OR_ZERO_SAFE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (!cur || !new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) case FTR_HIGHER_SAFE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) ret = new > cur ? new : cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) static void __init sort_ftr_regs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) for (i = 0; i < ARRAY_SIZE(arm64_ftr_regs); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) const struct arm64_ftr_reg *ftr_reg = arm64_ftr_regs[i].reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) const struct arm64_ftr_bits *ftr_bits = ftr_reg->ftr_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) unsigned int j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * Features here must be sorted in descending order with respect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * to their shift values and should not overlap with each other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) for (; ftr_bits->width != 0; ftr_bits++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) unsigned int width = ftr_reg->ftr_bits[j].width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) unsigned int shift = ftr_reg->ftr_bits[j].shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) unsigned int prev_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) WARN((shift + width) > 64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) "%s has invalid feature at shift %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) ftr_reg->name, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * Skip the first feature. There is nothing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * compare against for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (j == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) prev_shift = ftr_reg->ftr_bits[j - 1].shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) WARN((shift + width) > prev_shift,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) "%s has feature overlap at shift %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) ftr_reg->name, shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * Skip the first register. There is nothing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * compare against for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) if (i == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * Registers here must be sorted in ascending order with respect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * to sys_id for subsequent binary search in get_arm64_ftr_reg()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * to work correctly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * Initialise the CPU feature register from Boot CPU values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * Also initiliases the strict_mask for the register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * Any bits that are not covered by an arm64_ftr_bits entry are considered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * RES0 for the system-wide value, and must strictly match.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) static void init_cpu_ftr_reg(u32 sys_reg, u64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) u64 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) u64 strict_mask = ~0x0ULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) u64 user_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) u64 valid_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) const struct arm64_ftr_bits *ftrp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (!reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) u64 ftr_mask = arm64_ftr_mask(ftrp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) s64 ftr_new = arm64_ftr_value(ftrp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) s64 ftr_ovr = arm64_ftr_value(ftrp, reg->override->val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if ((ftr_mask & reg->override->mask) == ftr_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) s64 tmp = arm64_ftr_safe_value(ftrp, ftr_ovr, ftr_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) char *str = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (ftr_ovr != tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /* Unsafe, remove the override */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) reg->override->mask &= ~ftr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) reg->override->val &= ~ftr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) tmp = ftr_ovr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) str = "ignoring override";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) } else if (ftr_new != tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /* Override was valid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) ftr_new = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) str = "forced";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) } else if (ftr_ovr == tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) /* Override was the safe value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) str = "already set";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) if (str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) pr_warn("%s[%d:%d]: %s to %llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) reg->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) ftrp->shift + ftrp->width - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) ftrp->shift, str, tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) val = arm64_ftr_set_value(ftrp, val, ftr_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) valid_mask |= ftr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (!ftrp->strict)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) strict_mask &= ~ftr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (ftrp->visible)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) user_mask |= ftr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) reg->user_val = arm64_ftr_set_value(ftrp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) reg->user_val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) ftrp->safe_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) val &= valid_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) reg->sys_val = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) reg->strict_mask = strict_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) reg->user_mask = user_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) extern const struct arm64_cpu_capabilities arm64_errata[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static const struct arm64_cpu_capabilities arm64_features[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) static void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) for (; caps->matches; caps++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (WARN(caps->capability >= ARM64_NCAPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) "Invalid capability %d\n", caps->capability))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (WARN(cpu_hwcaps_ptrs[caps->capability],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) "Duplicate entry for capability %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) caps->capability))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) cpu_hwcaps_ptrs[caps->capability] = caps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) static void __init init_cpu_hwcaps_indirect_list(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) init_cpu_hwcaps_indirect_list_from_array(arm64_features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) init_cpu_hwcaps_indirect_list_from_array(arm64_errata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) static void __init setup_boot_cpu_capabilities(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) static void init_32bit_cpu_features(struct cpuinfo_32bit *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) void __init init_cpu_features(struct cpuinfo_arm64 *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) /* Before we start using the tables, make sure it is sorted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) sort_ftr_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) init_32bit_cpu_features(&info->aarch32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) sve_init_vq_map();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * Initialize the indirect array of CPU hwcaps capabilities pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * before we handle the boot CPU below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) init_cpu_hwcaps_indirect_list();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * Detect and enable early CPU capabilities based on the boot CPU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * after we have initialised the CPU feature infrastructure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) setup_boot_cpu_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) const struct arm64_ftr_bits *ftrp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) s64 ftr_new = arm64_ftr_value(ftrp, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) if (ftr_cur == ftr_new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) /* Find a safe value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (!regp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) update_cpu_ftr_reg(regp, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) if ((boot & regp->strict_mask) == (val & regp->strict_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) regp->name, boot, cpu, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) static void relax_cpu_ftr_reg(u32 sys_id, int field)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) const struct arm64_ftr_bits *ftrp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) if (!regp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) for (ftrp = regp->ftr_bits; ftrp->width; ftrp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (ftrp->shift == field) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) regp->strict_mask &= ~arm64_ftr_mask(ftrp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) /* Bogus field? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) WARN_ON(!ftrp->width);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) static void update_mismatched_32bit_el0_cpu_features(struct cpuinfo_arm64 *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) struct cpuinfo_arm64 *boot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) static bool boot_cpu_32bit_regs_overridden = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) boot->aarch32 = info->aarch32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) init_32bit_cpu_features(&boot->aarch32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) boot_cpu_32bit_regs_overridden = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) struct cpuinfo_32bit *boot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) int taint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * If we don't have AArch32 at EL1, then relax the strictness of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * EL1-dependent register fields to avoid spurious sanity check fails.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (!id_aa64pfr0_32bit_el1(pfr0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_SMC_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRT_FRAC_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SEC_FRAC_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_VIRTUALIZATION_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_SECURITY_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_PROGMOD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) info->reg_id_dfr0, boot->reg_id_dfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) taint |= check_update_ftr_reg(SYS_ID_DFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) info->reg_id_dfr1, boot->reg_id_dfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) info->reg_id_isar0, boot->reg_id_isar0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) info->reg_id_isar1, boot->reg_id_isar1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) info->reg_id_isar2, boot->reg_id_isar2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) info->reg_id_isar3, boot->reg_id_isar3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) info->reg_id_isar4, boot->reg_id_isar4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) info->reg_id_isar5, boot->reg_id_isar5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) taint |= check_update_ftr_reg(SYS_ID_ISAR6_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) info->reg_id_isar6, boot->reg_id_isar6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * ACTLR formats could differ across CPUs and therefore would have to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * be trapped for virtualization anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) info->reg_id_mmfr0, boot->reg_id_mmfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) info->reg_id_mmfr1, boot->reg_id_mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) info->reg_id_mmfr2, boot->reg_id_mmfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) info->reg_id_mmfr3, boot->reg_id_mmfr3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) taint |= check_update_ftr_reg(SYS_ID_MMFR4_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) info->reg_id_mmfr4, boot->reg_id_mmfr4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) taint |= check_update_ftr_reg(SYS_ID_MMFR5_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) info->reg_id_mmfr5, boot->reg_id_mmfr5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) info->reg_id_pfr0, boot->reg_id_pfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) info->reg_id_pfr1, boot->reg_id_pfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) taint |= check_update_ftr_reg(SYS_ID_PFR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) info->reg_id_pfr2, boot->reg_id_pfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) info->reg_mvfr0, boot->reg_mvfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) info->reg_mvfr1, boot->reg_mvfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) info->reg_mvfr2, boot->reg_mvfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) return taint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * Update system wide CPU feature registers with the values from a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * non-boot CPU. Also performs SANITY checks to make sure that there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * aren't any insane variations from that of the boot CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) void update_cpu_features(int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct cpuinfo_arm64 *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) struct cpuinfo_arm64 *boot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) int taint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) * The kernel can handle differing I-cache policies, but otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) * caches should look identical. Userspace JITs will make use of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) * *minLine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) info->reg_ctr, boot->reg_ctr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) * Userspace may perform DC ZVA instructions. Mismatched block sizes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * could result in too much or too little memory being zeroed if a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * process is preempted and migrated between CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) info->reg_dczid, boot->reg_dczid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) /* If different, timekeeping will be broken (especially with KVM) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) info->reg_cntfrq, boot->reg_cntfrq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * The kernel uses self-hosted debug features and expects CPUs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * support identical debug features. We presently need CTX_CMPs, WRPs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * and BRPs to be identical.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) * ID_AA64DFR1 is currently RES0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * Even in big.LITTLE, processors should be identical instruction-set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * wise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * Differing PARange support is fine as long as all peripherals and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) * memory are mapped within the minimum PARange of all CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * Linux should not care about secure memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) info->reg_zcr, boot->reg_zcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) /* Probe vector lengths, unless we already gave up on SVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) !system_capabilities_finalized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) sve_update_vq_map();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) * If we don't have AArch32 at all then skip the checks entirely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) * as the register values may be UNKNOWN and we're not going to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * using them for anything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * This relies on a sanitised view of the AArch64 ID registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) update_mismatched_32bit_el0_cpu_features(info, boot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) taint |= update_32bit_cpu_features(cpu, &info->aarch32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) &boot->aarch32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * Mismatched CPU features are a recipe for disaster. Don't even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * pretend to support them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) if (taint) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) pr_warn_once("Unsupported CPU feature variation detected.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) u64 read_sanitised_ftr_reg(u32 id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (!regp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return regp->sys_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) #define read_sysreg_case(r) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) case r: val = read_sysreg_s(r); break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * Read the system register on the current CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) u64 __read_sysreg_by_encoding(u32 sys_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) struct arm64_ftr_reg *regp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) switch (sys_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) read_sysreg_case(SYS_ID_PFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) read_sysreg_case(SYS_ID_PFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) read_sysreg_case(SYS_ID_PFR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) read_sysreg_case(SYS_ID_DFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) read_sysreg_case(SYS_ID_DFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) read_sysreg_case(SYS_ID_MMFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) read_sysreg_case(SYS_ID_MMFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) read_sysreg_case(SYS_ID_MMFR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) read_sysreg_case(SYS_ID_MMFR3_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) read_sysreg_case(SYS_ID_MMFR4_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) read_sysreg_case(SYS_ID_MMFR5_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) read_sysreg_case(SYS_ID_ISAR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) read_sysreg_case(SYS_ID_ISAR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) read_sysreg_case(SYS_ID_ISAR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) read_sysreg_case(SYS_ID_ISAR3_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) read_sysreg_case(SYS_ID_ISAR4_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) read_sysreg_case(SYS_ID_ISAR5_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) read_sysreg_case(SYS_ID_ISAR6_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) read_sysreg_case(SYS_MVFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) read_sysreg_case(SYS_MVFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) read_sysreg_case(SYS_MVFR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) read_sysreg_case(SYS_ID_AA64PFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) read_sysreg_case(SYS_ID_AA64PFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) read_sysreg_case(SYS_ID_AA64ZFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) read_sysreg_case(SYS_ID_AA64DFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) read_sysreg_case(SYS_ID_AA64DFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) read_sysreg_case(SYS_ID_AA64ISAR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) read_sysreg_case(SYS_CNTFRQ_EL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) read_sysreg_case(SYS_CTR_EL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) read_sysreg_case(SYS_DCZID_EL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) regp = get_arm64_ftr_reg(sys_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (regp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) val &= ~regp->override->mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) val |= (regp->override->val & regp->override->mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) #include <linux/irqchip/arm-gic-v3.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) return val >= entry->min_field_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) if (scope == SCOPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) val = read_sanitised_ftr_reg(entry->sys_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) val = __read_sysreg_by_encoding(entry->sys_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return feature_matches(val, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) const struct cpumask *system_32bit_el0_cpumask(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) if (!system_supports_32bit_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) return cpu_none_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) return cpu_32bit_el0_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) return cpu_possible_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) EXPORT_SYMBOL_GPL(system_32bit_el0_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) static int __init parse_32bit_el0_param(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) allow_mismatched_32bit_el0 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) early_param("allow_mismatched_32bit_el0", parse_32bit_el0_param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) static ssize_t aarch32_el0_show(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) const struct cpumask *mask = system_32bit_el0_cpumask();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) static const DEVICE_ATTR_RO(aarch32_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) static int __init aarch32_el0_sysfs_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (!allow_mismatched_32bit_el0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) return device_create_file(cpu_subsys.dev_root, &dev_attr_aarch32_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) device_initcall(aarch32_el0_sysfs_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) if (!has_cpuid_feature(entry, scope))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) return allow_mismatched_32bit_el0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) if (scope == SCOPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) pr_info("detected: 32-bit EL0 Support\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) bool has_sre;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (!has_cpuid_feature(entry, scope))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) has_sre = gic_enable_sre();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) if (!has_sre)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) pr_warn_once("%s present but disabled by higher exception level\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) entry->desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) return has_sre;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) u32 midr = read_cpuid_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) /* Cavium ThunderX pass 1.x and 2.x */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) return midr_is_cpu_model_range(midr, MIDR_THUNDERX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) MIDR_CPU_VAR_REV(0, 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) return cpuid_feature_extract_signed_field(pfr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) ID_AA64PFR0_FP_SHIFT) < 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) u64 ctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) if (scope == SCOPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) ctr = arm64_ftr_reg_ctrel0.sys_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ctr = read_cpuid_effective_cachetype();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) return ctr & BIT(CTR_IDC_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) * to the CTR_EL0 on this CPU and emulate it with the real/safe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) * value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (!(read_cpuid_cachetype() & BIT(CTR_IDC_SHIFT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) u64 ctr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) if (scope == SCOPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) ctr = arm64_ftr_reg_ctrel0.sys_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) ctr = read_cpuid_cachetype();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) return ctr & BIT(CTR_DIC_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) static bool __maybe_unused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * may share TLB entries with a CPU stuck in the crashed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (is_kdump_kernel())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return has_cpuid_feature(entry, scope);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) * This check is triggered during the early boot before the cpufeature
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) * is initialised. Checking the status on the local CPU allows the boot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) * CPU to detect the need for non-global mappings and thus avoiding a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) * pagetable re-write after all the CPUs are booted. This check will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) * anyway run on individual CPUs, allowing us to get the consistent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * state once the SMP CPUs are up and thus make the switch to non-global
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) * mappings if required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) bool kaslr_requires_kpti(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) * E0PD does a similar job to KPTI so can be used instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) * where available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) if (IS_ENABLED(CONFIG_ARM64_E0PD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) if (cpuid_feature_extract_unsigned_field(mmfr2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) ID_AA64MMFR2_E0PD_SHIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) * Systems affected by Cavium erratum 24756 are incompatible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * with KPTI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) extern const struct midr_range cavium_erratum_27456_cpus[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) if (is_midr_in_range_list(read_cpuid_id(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) cavium_erratum_27456_cpus))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) return kaslr_offset() > 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) static bool __meltdown_safe = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) /* List of CPUs that are not vulnerable and don't need KPTI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) static const struct midr_range kpti_safe_list[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_GOLD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) { /* sentinel */ }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) char const *str = "kpti command line option";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) bool meltdown_safe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) meltdown_safe = is_midr_in_range_list(read_cpuid_id(), kpti_safe_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) /* Defer to CPU feature registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (has_cpuid_feature(entry, scope))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) meltdown_safe = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) if (!meltdown_safe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) __meltdown_safe = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) * For reasons that aren't entirely clear, enabling KPTI on Cavium
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * ThunderX leads to apparent I-cache corruption of kernel text, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * ends as well as you might imagine. Don't even try.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) str = "ARM64_WORKAROUND_CAVIUM_27456";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) __kpti_forced = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) /* Useful for KASLR robustness */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (kaslr_requires_kpti()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) if (!__kpti_forced) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) str = "KASLR";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) __kpti_forced = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) if (cpu_mitigations_off() && !__kpti_forced) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) str = "mitigations=off";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) __kpti_forced = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) pr_info_once("kernel page table isolation disabled by kernel configuration\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) /* Forced? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) if (__kpti_forced) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) pr_info_once("kernel page table isolation forced %s by %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) __kpti_forced > 0 ? "ON" : "OFF", str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) return __kpti_forced > 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) return !meltdown_safe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) static void __nocfi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) typedef void (kpti_remap_fn)(int, int, phys_addr_t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) extern kpti_remap_fn idmap_kpti_install_ng_mappings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) kpti_remap_fn *remap_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) if (__this_cpu_read(this_cpu_vector) == vectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) __this_cpu_write(this_cpu_vector, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) * We don't need to rewrite the page-tables if either we've done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) * it already or we have KASLR enabled and therefore have not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) * created any global mappings at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) if (arm64_use_ng_mappings)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) remap_fn = (void *)__pa_function(idmap_kpti_install_ng_mappings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) cpu_install_idmap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) cpu_uninstall_idmap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) if (!cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) arm64_use_ng_mappings = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) static int __init parse_kpti(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) bool enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) int ret = strtobool(str, &enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) __kpti_forced = enabled ? 1 : -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) early_param("kpti", parse_kpti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) #ifdef CONFIG_ARM64_HW_AFDBM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) static inline void __cpu_enable_hw_dbm(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) write_sysreg(tcr, tcr_el1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) local_flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) static bool cpu_has_broken_dbm(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) /* List of CPUs which have broken DBM support. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) static const struct midr_range cpus[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) #ifdef CONFIG_ARM64_ERRATUM_1024718
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) /* Kryo4xx Silver (rdpe => r1p0) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) #ifdef CONFIG_ARM64_ERRATUM_2051678
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) return is_midr_in_range_list(read_cpuid_id(), cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) !cpu_has_broken_dbm();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) if (cpu_can_use_dbm(cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) __cpu_enable_hw_dbm();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) static bool detected = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * DBM is a non-conflicting feature. i.e, the kernel can safely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) * run a mix of CPUs with and without the feature. So, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) * unconditionally enable the capability to allow any late CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) * to use the feature. We only enable the control bits on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) * CPU, if it actually supports.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) * We have to make sure we print the "feature" detection only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) * when at least one CPU actually uses it. So check if this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) * can actually use it and print the message exactly once.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) * This is safe as all CPUs (including secondary CPUs - due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) * LOCAL_CPU scope - and the hotplugged CPUs - via verification)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) * goes through the "matches" check exactly once. Also if a CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) * matches the criteria, it is guaranteed that the CPU will turn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) * the DBM on, as the capability is unconditionally enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) if (!detected && cpu_can_use_dbm(cap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) detected = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) pr_info("detected: Hardware dirty bit management\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) #ifdef CONFIG_ARM64_AMU_EXTN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) * The "amu_cpus" cpumask only signals that the CPU implementation for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) * information regarding all the events that it supports. When a CPU bit is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) * set in the cpumask, the user of this feature can only rely on the presence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) * of the 4 fixed counters for that CPU. But this does not guarantee that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) * counters are enabled or access to these counters is enabled by code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) * executed at higher exception levels (firmware).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) static struct cpumask amu_cpus __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) bool cpu_has_amu_feat(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) return cpumask_test_cpu(cpu, &amu_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) /* Initialize the use of AMU counters for frequency invariance */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) extern void init_cpu_freq_invariance_counters(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) cpumask_set_cpu(smp_processor_id(), &amu_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) init_cpu_freq_invariance_counters();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) static bool has_amu(const struct arm64_cpu_capabilities *cap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) * The AMU extension is a non-conflicting feature: the kernel can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * safely run a mix of CPUs with and without support for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) * activity monitors extension. Therefore, unconditionally enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) * the capability to allow any late CPU to use the feature.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) * With this feature unconditionally enabled, the cpu_enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) * function will be called for all CPUs that match the criteria,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) * including secondary and hotplugged, marking this feature as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) * present on that respective CPU. The enable function will also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * print a detection message.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) #ifdef CONFIG_ARM64_VHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) return is_kernel_in_hyp_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) * Copy register values that aren't redirected by hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) * Before code patching, we only set tpidr_el1, all CPUs need to copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) * this value to tpidr_el2 before we patch the code. Once we've done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * do anything here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) u64 val = read_sysreg_s(SYS_CLIDR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) /* Check that CLIDR_EL1.LOU{U,IS} are both 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) WARN_ON(val & (7 << 27 | 7 << 21));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) #ifdef CONFIG_ARM64_PAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) * We modify PSTATE. This won't work from irq context as the PSTATE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) * is discarded once we return from the exception.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) WARN_ON_ONCE(in_interrupt());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) set_pstate_pan(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) #endif /* CONFIG_ARM64_PAN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) #ifdef CONFIG_ARM64_RAS_EXTN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) /* Firmware may have left a deferred SError in this register. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) write_sysreg_s(0, SYS_DISR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) #endif /* CONFIG_ARM64_RAS_EXTN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) #ifdef CONFIG_ARM64_PTR_AUTH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) int boot_val, sec_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) /* We don't expect to be called with SCOPE_SYSTEM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) WARN_ON(scope == SCOPE_SYSTEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) * The ptr-auth feature levels are not intercompatible with lower
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) * levels. Hence we must match ptr-auth feature level of the secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) * CPUs with that of the boot CPU. The level of boot cpu is fetched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) * from the sanitised register whereas direct register read is done for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) * the secondary CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) * The sanitised feature state is guaranteed to match that of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) * boot CPU as a mismatched secondary CPU is parked before it gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) * a chance to update the state, with the capability.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) entry->field_pos, entry->sign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) if (scope & SCOPE_BOOT_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) return boot_val >= entry->min_field_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) /* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) entry->field_pos, entry->sign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) return sec_val == boot_val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) return has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH], scope) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) return __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) #endif /* CONFIG_ARM64_PTR_AUTH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) #ifdef CONFIG_ARM64_E0PD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) if (this_cpu_has_cap(ARM64_HAS_E0PD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) sysreg_clear_set(tcr_el1, 0, TCR_E0PD1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) #endif /* CONFIG_ARM64_E0PD */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) #ifdef CONFIG_ARM64_PSEUDO_NMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) static bool enable_pseudo_nmi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) static int __init early_enable_pseudo_nmi(char *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) return strtobool(p, &enable_pseudo_nmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) early_param("irqchip.gicv3_pseudo_nmi", early_enable_pseudo_nmi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) return enable_pseudo_nmi && has_useable_gicv3_cpuif(entry, scope);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) #ifdef CONFIG_ARM64_BTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) static void bti_enable(const struct arm64_cpu_capabilities *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) * Use of X16/X17 for tail-calls and trampolines that jump to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) * function entry points using BR is a requirement for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) * marking binaries with GNU_PROPERTY_AARCH64_FEATURE_1_BTI.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) * So, be strict and forbid other BRs using other registers to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) * jump onto a PACIxSP instruction:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) #endif /* CONFIG_ARM64_BTI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) #ifdef CONFIG_ARM64_MTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) * Clear the tags in the zero page. This needs to be done via the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) * linear map which has the Tagged attribute.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) if (!test_and_set_bit(PG_mte_tagged, &ZERO_PAGE(0)->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) mte_clear_page_tags(lm_alias(empty_zero_page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) kasan_init_hw_tags_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) #endif /* CONFIG_ARM64_MTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) #ifdef CONFIG_KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (kvm_get_mode() != KVM_MODE_PROTECTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) if (is_kernel_in_hyp_mode()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) pr_warn("Protected KVM not available with VHE\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) #endif /* CONFIG_KVM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) /* Internal helper functions to match cpu capability type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) static const struct arm64_cpu_capabilities arm64_features[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) .desc = "GIC system register CPU interface",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) .matches = has_useable_gicv3_cpuif,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) .field_pos = ID_AA64PFR0_GIC_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) #ifdef CONFIG_ARM64_PAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) .desc = "Privileged Access Never",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) .capability = ARM64_HAS_PAN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) .sys_reg = SYS_ID_AA64MMFR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) .field_pos = ID_AA64MMFR1_PAN_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) .cpu_enable = cpu_enable_pan,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) #endif /* CONFIG_ARM64_PAN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) #ifdef CONFIG_ARM64_LSE_ATOMICS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) .desc = "LSE atomic instructions",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) .capability = ARM64_HAS_LSE_ATOMICS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) .sys_reg = SYS_ID_AA64ISAR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) .min_field_value = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) #endif /* CONFIG_ARM64_LSE_ATOMICS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) .desc = "Software prefetching using PRFM",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) .capability = ARM64_HAS_NO_HW_PREFETCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) .matches = has_no_hw_prefetch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) #ifdef CONFIG_ARM64_UAO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) .desc = "User Access Override",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) .capability = ARM64_HAS_UAO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) .sys_reg = SYS_ID_AA64MMFR2_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) .field_pos = ID_AA64MMFR2_UAO_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) * We rely on stop_machine() calling uao_thread_switch() to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) * UAO immediately after patching.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) #endif /* CONFIG_ARM64_UAO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) #ifdef CONFIG_ARM64_PAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) .capability = ARM64_ALT_PAN_NOT_UAO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) .matches = cpufeature_pan_not_uao,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) #endif /* CONFIG_ARM64_PAN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) #ifdef CONFIG_ARM64_VHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) .desc = "Virtualization Host Extensions",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) .capability = ARM64_HAS_VIRT_HOST_EXTN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) .matches = runs_at_el2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) .cpu_enable = cpu_copy_el2regs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) #endif /* CONFIG_ARM64_VHE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) .matches = has_32bit_el0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) .field_pos = ID_AA64PFR0_EL0_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) #ifdef CONFIG_KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) .desc = "32-bit EL1 Support",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) .capability = ARM64_HAS_32BIT_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) .field_pos = ID_AA64PFR0_EL1_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) .min_field_value = ID_AA64PFR0_EL1_32BIT_64BIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) .desc = "Protected KVM",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) .capability = ARM64_KVM_PROTECTED_MODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) .matches = is_kvm_protected_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) .desc = "Kernel page table isolation (KPTI)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) .capability = ARM64_UNMAP_KERNEL_AT_EL0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) * The ID feature fields below are used to indicate that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) * more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) .field_pos = ID_AA64PFR0_CSV3_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) .matches = unmap_kernel_at_el0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) .cpu_enable = kpti_install_ng_mappings,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) /* FP/SIMD is not implemented */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) .capability = ARM64_HAS_NO_FPSIMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) .min_field_value = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) .matches = has_no_fpsimd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) #ifdef CONFIG_ARM64_PMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) .desc = "Data cache clean to Point of Persistence",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) .capability = ARM64_HAS_DCPOP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) .field_pos = ID_AA64ISAR1_DPB_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) .desc = "Data cache clean to Point of Deep Persistence",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) .capability = ARM64_HAS_DCPODP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) .field_pos = ID_AA64ISAR1_DPB_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) .min_field_value = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) #ifdef CONFIG_ARM64_SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) .desc = "Scalable Vector Extension",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) .capability = ARM64_SVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) .field_pos = ID_AA64PFR0_SVE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) .min_field_value = ID_AA64PFR0_SVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) .cpu_enable = sve_kernel_enable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) #endif /* CONFIG_ARM64_SVE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) #ifdef CONFIG_ARM64_RAS_EXTN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) .desc = "RAS Extension Support",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) .capability = ARM64_HAS_RAS_EXTN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) .field_pos = ID_AA64PFR0_RAS_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) .min_field_value = ID_AA64PFR0_RAS_V1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) .cpu_enable = cpu_clear_disr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) #endif /* CONFIG_ARM64_RAS_EXTN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) #ifdef CONFIG_ARM64_AMU_EXTN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) * The feature is enabled by default if CONFIG_ARM64_AMU_EXTN=y.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) * Therefore, don't provide .desc as we don't want the detection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) * message to be shown until at least one CPU is detected to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) * support the feature.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) .capability = ARM64_HAS_AMU_EXTN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) .matches = has_amu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) .field_pos = ID_AA64PFR0_AMU_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) .min_field_value = ID_AA64PFR0_AMU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) .cpu_enable = cpu_amu_enable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) #endif /* CONFIG_ARM64_AMU_EXTN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) .desc = "Data cache clean to the PoU not required for I/D coherence",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) .capability = ARM64_HAS_CACHE_IDC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) .matches = has_cache_idc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) .cpu_enable = cpu_emulate_effective_ctr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) .desc = "Instruction cache invalidation not required for I/D coherence",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) .capability = ARM64_HAS_CACHE_DIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) .matches = has_cache_dic,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) .desc = "Stage-2 Force Write-Back",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) .capability = ARM64_HAS_STAGE2_FWB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) .sys_reg = SYS_ID_AA64MMFR2_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) .field_pos = ID_AA64MMFR2_FWB_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) .cpu_enable = cpu_has_fwb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) .desc = "ARMv8.4 Translation Table Level",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) .capability = ARM64_HAS_ARMv8_4_TTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) .sys_reg = SYS_ID_AA64MMFR2_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) .field_pos = ID_AA64MMFR2_TTL_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) .desc = "TLB range maintenance instructions",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) .capability = ARM64_HAS_TLB_RANGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) .sys_reg = SYS_ID_AA64ISAR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) .field_pos = ID_AA64ISAR0_TLB_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) .min_field_value = ID_AA64ISAR0_TLB_RANGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) #ifdef CONFIG_ARM64_HW_AFDBM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) * Since we turn this on always, we don't want the user to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) * think that the feature is available when it may not be.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) * So hide the description.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) * .desc = "Hardware pagetable Dirty Bit Management",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) .capability = ARM64_HW_DBM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) .sys_reg = SYS_ID_AA64MMFR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) .field_pos = ID_AA64MMFR1_HADBS_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) .min_field_value = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) .matches = has_hw_dbm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) .cpu_enable = cpu_enable_hw_dbm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) .desc = "CRC32 instructions",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) .capability = ARM64_HAS_CRC32,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) .sys_reg = SYS_ID_AA64ISAR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) .field_pos = ID_AA64ISAR0_CRC32_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) .desc = "Speculative Store Bypassing Safe (SSBS)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) .capability = ARM64_SSBS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) .sys_reg = SYS_ID_AA64PFR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) .field_pos = ID_AA64PFR1_SSBS_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) #ifdef CONFIG_ARM64_CNP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) .desc = "Common not Private translations",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) .capability = ARM64_HAS_CNP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) .matches = has_useable_cnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) .sys_reg = SYS_ID_AA64MMFR2_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) .field_pos = ID_AA64MMFR2_CNP_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) .cpu_enable = cpu_enable_cnp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) .desc = "Speculation barrier (SB)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) .capability = ARM64_HAS_SB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) .field_pos = ID_AA64ISAR1_SB_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) #ifdef CONFIG_ARM64_PTR_AUTH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) .desc = "Address authentication (architected algorithm)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) .capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) .field_pos = ID_AA64ISAR1_APA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) .min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) .matches = has_address_auth_cpucap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) .desc = "Address authentication (IMP DEF algorithm)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) .field_pos = ID_AA64ISAR1_API_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) .min_field_value = ID_AA64ISAR1_API_IMP_DEF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) .matches = has_address_auth_cpucap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) .capability = ARM64_HAS_ADDRESS_AUTH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) .matches = has_address_auth_metacap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) .desc = "Generic authentication (architected algorithm)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) .capability = ARM64_HAS_GENERIC_AUTH_ARCH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) .field_pos = ID_AA64ISAR1_GPA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) .min_field_value = ID_AA64ISAR1_GPA_ARCHITECTED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) .desc = "Generic authentication (IMP DEF algorithm)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) .capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) .field_pos = ID_AA64ISAR1_GPI_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) .min_field_value = ID_AA64ISAR1_GPI_IMP_DEF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) .capability = ARM64_HAS_GENERIC_AUTH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) .matches = has_generic_auth,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) #endif /* CONFIG_ARM64_PTR_AUTH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) #ifdef CONFIG_ARM64_PSEUDO_NMI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) * Depends on having GICv3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) .desc = "IRQ priority masking",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) .capability = ARM64_HAS_IRQ_PRIO_MASKING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) .matches = can_use_gic_priorities,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) .sys_reg = SYS_ID_AA64PFR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) .field_pos = ID_AA64PFR0_GIC_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) #ifdef CONFIG_ARM64_E0PD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) .desc = "E0PD",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) .capability = ARM64_HAS_E0PD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) .sys_reg = SYS_ID_AA64MMFR2_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) .field_pos = ID_AA64MMFR2_E0PD_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) .cpu_enable = cpu_enable_e0pd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) #ifdef CONFIG_ARCH_RANDOM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) .desc = "Random Number Generator",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) .capability = ARM64_HAS_RNG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) .sys_reg = SYS_ID_AA64ISAR0_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) .field_pos = ID_AA64ISAR0_RNDR_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) #ifdef CONFIG_ARM64_BTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) .desc = "Branch Target Identification",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) .capability = ARM64_BTI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) #ifdef CONFIG_ARM64_BTI_KERNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) .cpu_enable = bti_enable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) .sys_reg = SYS_ID_AA64PFR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) .field_pos = ID_AA64PFR1_BT_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) .min_field_value = ID_AA64PFR1_BT_BTI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) #ifdef CONFIG_ARM64_MTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) .desc = "Memory Tagging Extension",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) .capability = ARM64_MTE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) .sys_reg = SYS_ID_AA64PFR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) .field_pos = ID_AA64PFR1_MTE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) .min_field_value = ID_AA64PFR1_MTE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) .cpu_enable = cpu_enable_mte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) #endif /* CONFIG_ARM64_MTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) .desc = "RCpc load-acquire (LDAPR)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) .capability = ARM64_HAS_LDAPR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) .type = ARM64_CPUCAP_SYSTEM_FEATURE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) .sys_reg = SYS_ID_AA64ISAR1_EL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) .sign = FTR_UNSIGNED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) .field_pos = ID_AA64ISAR1_LRCPC_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) .matches = has_cpuid_feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) .min_field_value = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) #define HWCAP_CPUID_MATCH(reg, field, s, min_value) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) .matches = has_cpuid_feature, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) .sys_reg = reg, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) .field_pos = field, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) .sign = s, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) .min_field_value = min_value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) #define __HWCAP_CAP(name, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) .desc = name, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) .type = ARM64_CPUCAP_SYSTEM_FEATURE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) .hwcap_type = cap_type, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) .hwcap = cap, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) #define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) __HWCAP_CAP(#cap, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) HWCAP_CPUID_MATCH(reg, field, s, min_value) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) #define HWCAP_MULTI_CAP(list, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) __HWCAP_CAP(#cap, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) .matches = cpucap_multi_entry_cap_matches, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) .match_list = list, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) #define HWCAP_CAP_MATCH(match, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) __HWCAP_CAP(#cap, cap_type, cap) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) .matches = match, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) #ifdef CONFIG_ARM64_PTR_AUTH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_APA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) FTR_UNSIGNED, ID_AA64ISAR1_APA_ARCHITECTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) FTR_UNSIGNED, ID_AA64ISAR1_API_IMP_DEF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPA_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) FTR_UNSIGNED, ID_AA64ISAR1_GPA_ARCHITECTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPI_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) FTR_UNSIGNED, ID_AA64ISAR1_GPI_IMP_DEF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_PMULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_SHA512),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_CRC32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ATOMICS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SHA3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SM4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RNDR_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RNG),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DCPOP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_DCPODP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_JSCVT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FCMA),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_LRCPC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FRINTTS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FRINT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_SB),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_BF16_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_BF16),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DGH_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DGH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_I8MM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_I8MM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) #ifdef CONFIG_ARM64_SVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, KERNEL_HWCAP_SVE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SVEVER_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SVEVER_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_AES, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_AES_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_AES_PMULL, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BITPERM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_BITPERM, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_BF16_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_BF16, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SHA3_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SHA3, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_SM4_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_SM4, CAP_HWCAP, KERNEL_HWCAP_SVESM4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_I8MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_I8MM, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F32MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_F32MM, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_F64MM_SHIFT, FTR_UNSIGNED, ID_AA64ZFR0_F64MM, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) #ifdef CONFIG_ARM64_BTI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) #ifdef CONFIG_ARM64_PTR_AUTH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) #ifdef CONFIG_ARM64_MTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) #endif /* CONFIG_ARM64_MTE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) * Check that all of MVFR1_EL1.{SIMDSP, SIMDInt, SIMDLS} are available,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) * in line with that of arm32 as in vfp_init(). We make sure that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) * check is future proof, by making sure value is non-zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) u32 mvfr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) if (scope == SCOPE_SYSTEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) mvfr1 = read_sanitised_ftr_reg(SYS_MVFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDSP_SHIFT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDINT_SHIFT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_SIMDLS_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) HWCAP_CAP(SYS_MVFR1_EL1, MVFR1_SIMDFMAC_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) /* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) HWCAP_CAP(SYS_MVFR0_EL1, MVFR0_FPDP_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) {},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) switch (cap->hwcap_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) case CAP_HWCAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) cpu_set_feature(cap->hwcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) case CAP_COMPAT_HWCAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) compat_elf_hwcap |= (u32)cap->hwcap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) case CAP_COMPAT_HWCAP2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) compat_elf_hwcap2 |= (u32)cap->hwcap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) /* Check if we have a particular HWCAP enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) bool rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) switch (cap->hwcap_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) case CAP_HWCAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) rc = cpu_have_feature(cap->hwcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) #ifdef CONFIG_COMPAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) case CAP_COMPAT_HWCAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) case CAP_COMPAT_HWCAP2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) rc = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) /* We support emulation of accesses to CPU ID feature registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) cpu_set_named_feature(CPUID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) for (; hwcaps->matches; hwcaps++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) cap_set_elf_hwcap(hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) static void update_cpu_capabilities(u16 scope_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) const struct arm64_cpu_capabilities *caps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) for (i = 0; i < ARM64_NCAPS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) caps = cpu_hwcaps_ptrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) if (!caps || !(caps->type & scope_mask) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) cpus_have_cap(caps->capability) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) !caps->matches(caps, cpucap_default_scope(caps)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) if (caps->desc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) pr_info("detected: %s\n", caps->desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) cpus_set_cap(caps->capability);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) set_bit(caps->capability, boot_capabilities);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) * Enable all the available capabilities on this CPU. The capabilities
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) * with BOOT_CPU scope are handled separately and hence skipped here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) static int cpu_enable_non_boot_scope_capabilities(void *__unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) for_each_available_cap(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) if (WARN_ON(!cap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) if (!(cap->type & non_boot_scope))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) if (cap->cpu_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) cap->cpu_enable(cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) * Run through the enabled capabilities and enable() it on all active
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) * CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) static void __init enable_cpu_capabilities(u16 scope_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) const struct arm64_cpu_capabilities *caps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) bool boot_scope;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) for (i = 0; i < ARM64_NCAPS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) unsigned int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) caps = cpu_hwcaps_ptrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) if (!caps || !(caps->type & scope_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) num = caps->capability;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) if (!cpus_have_cap(num))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) /* Ensure cpus_have_const_cap(num) works */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) static_branch_enable(&cpu_hwcap_keys[num]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) if (boot_scope && caps->cpu_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) * Capabilities with SCOPE_BOOT_CPU scope are finalised
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) * before any secondary CPU boots. Thus, each secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) * will enable the capability as appropriate via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) * check_local_cpu_capabilities(). The only exception is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) * the boot CPU, for which the capability must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) * enabled here. This approach avoids costly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) * stop_machine() calls for this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) caps->cpu_enable(caps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) * For all non-boot scope capabilities, use stop_machine()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) * as it schedules the work allowing us to modify PSTATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) * instead of on_each_cpu() which uses an IPI, giving us a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) * PSTATE that disappears when we return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) if (!boot_scope)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) stop_machine(cpu_enable_non_boot_scope_capabilities,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) NULL, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) * Run through the list of capabilities to check for conflicts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) * If the system has already detected a capability, take necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) * action on this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) static void verify_local_cpu_caps(u16 scope_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) bool cpu_has_cap, system_has_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) const struct arm64_cpu_capabilities *caps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) for (i = 0; i < ARM64_NCAPS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) caps = cpu_hwcaps_ptrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) if (!caps || !(caps->type & scope_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) system_has_cap = cpus_have_cap(caps->capability);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) if (system_has_cap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) * Check if the new CPU misses an advertised feature,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) * which is not safe to miss.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) if (!cpu_has_cap && !cpucap_late_cpu_optional(caps))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) * We have to issue cpu_enable() irrespective of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) * whether the CPU has it or not, as it is enabeld
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) * system wide. It is upto the call back to take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) * appropriate action on this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) if (caps->cpu_enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) caps->cpu_enable(caps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) * Check if the CPU has this capability if it isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) * safe to have when the system doesn't.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) if (cpu_has_cap && !cpucap_late_cpu_permitted(caps))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) if (i < ARM64_NCAPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) smp_processor_id(), caps->capability,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) caps->desc, system_has_cap, cpu_has_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) if (cpucap_panic_on_conflict(caps))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) cpu_panic_kernel();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) cpu_die_early();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) * Check for CPU features that are used in early boot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) * based on the Boot CPU value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) static void check_early_cpu_features(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) verify_cpu_asid_bits();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) verify_local_cpu_caps(SCOPE_BOOT_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) __verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) for (; caps->matches; caps++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) pr_crit("CPU%d: missing HWCAP: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) smp_processor_id(), caps->desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) cpu_die_early();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) static void verify_local_elf_hwcaps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) __verify_local_elf_hwcaps(arm64_elf_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) __verify_local_elf_hwcaps(compat_elf_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) static void verify_sve_features(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) u64 zcr = read_zcr_features();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) unsigned int len = zcr & ZCR_ELx_LEN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) if (len < safe_len || sve_verify_vq_map()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) pr_crit("CPU%d: SVE: vector length support mismatch\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) cpu_die_early();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) /* Add checks on other ZCR bits here if necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) static void verify_hyp_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) u64 safe_mmfr1, mmfr0, mmfr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) int parange, ipa_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) unsigned int safe_vmid_bits, vmid_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) if (!IS_ENABLED(CONFIG_KVM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) safe_mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) /* Verify VMID bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) safe_vmid_bits = get_vmid_bits(safe_mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) vmid_bits = get_vmid_bits(mmfr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) if (vmid_bits < safe_vmid_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) pr_crit("CPU%d: VMID width mismatch\n", smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) cpu_die_early();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) /* Verify IPA range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) parange = cpuid_feature_extract_unsigned_field(mmfr0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) ID_AA64MMFR0_PARANGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) if (ipa_max < get_kvm_ipa_limit()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) pr_crit("CPU%d: IPA range mismatch\n", smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) cpu_die_early();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) * Run through the enabled system capabilities and enable() it on this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) * The capabilities were decided based on the available CPUs at the boot time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) * Any new CPU should match the system wide status of the capability. If the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) * new CPU doesn't have a capability which the system now has enabled, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) * cannot do anything to fix it up and could cause unexpected failures. So
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) * we park the CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) static void verify_local_cpu_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) * The capabilities with SCOPE_BOOT_CPU are checked from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) * check_early_cpu_features(), as they need to be verified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) * on all secondary CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) verify_local_elf_hwcaps();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) if (system_supports_sve())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) verify_sve_features();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) if (is_hyp_mode_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) verify_hyp_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) void check_local_cpu_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) * All secondary CPUs should conform to the early CPU features
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) * in use by the kernel based on boot CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) check_early_cpu_features();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) * If we haven't finalised the system capabilities, this CPU gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) * a chance to update the errata work arounds and local features.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) * Otherwise, this CPU should verify that it has all the system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) * advertised capabilities.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) if (!system_capabilities_finalized())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) update_cpu_capabilities(SCOPE_LOCAL_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) verify_local_cpu_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) static void __init setup_boot_cpu_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) /* Detect capabilities with either SCOPE_BOOT_CPU or SCOPE_LOCAL_CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) /* Enable the SCOPE_BOOT_CPU capabilities alone right away */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) enable_cpu_capabilities(SCOPE_BOOT_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) bool this_cpu_has_cap(unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) if (cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) return cap->matches(cap, SCOPE_LOCAL_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) * This helper function is used in a narrow window when,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) * - The system wide safe registers are set with all the SMP CPUs and,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) * - The SYSTEM_FEATURE cpu_hwcaps may not have been set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) * In all other cases cpus_have_{const_}cap() should be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) static bool __system_matches_cap(unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) if (n < ARM64_NCAPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) if (cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) return cap->matches(cap, SCOPE_SYSTEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) void cpu_set_feature(unsigned int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) WARN_ON(num >= MAX_CPU_FEATURES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) elf_hwcap |= BIT(num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) EXPORT_SYMBOL_GPL(cpu_set_feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) bool cpu_have_feature(unsigned int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) WARN_ON(num >= MAX_CPU_FEATURES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) return elf_hwcap & BIT(num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) EXPORT_SYMBOL_GPL(cpu_have_feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) unsigned long cpu_get_elf_hwcap(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) * We currently only populate the first 32 bits of AT_HWCAP. Please
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) * note that for userspace compatibility we guarantee that bits 62
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * and 63 will always be returned as 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) return lower_32_bits(elf_hwcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) unsigned long cpu_get_elf_hwcap2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) return upper_32_bits(elf_hwcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) static void __init setup_system_capabilities(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) * We have finalised the system-wide safe feature
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) * registers, finalise the capabilities that depend
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) * on it. Also enable all the available capabilities,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) * that are not enabled already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) update_cpu_capabilities(SCOPE_SYSTEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) void __init setup_cpu_features(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) u32 cwg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) setup_system_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) setup_elf_hwcaps(arm64_elf_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) if (system_supports_32bit_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) setup_elf_hwcaps(compat_elf_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) if (system_uses_ttbr0_pan())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) sve_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) minsigstksz_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) /* Advertise that we have computed the system capabilities */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) finalize_system_capabilities();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) * Check for sane CTR_EL0.CWG value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) cwg = cache_type_cwg();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) if (!cwg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) pr_warn("No Cache Writeback Granule information, assuming %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) ARCH_DMA_MINALIGN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) static int enable_mismatched_32bit_el0(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) static int lucky_winner = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) if (cpu_32bit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) cpumask_set_cpu(cpu, cpu_32bit_el0_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) if (cpumask_test_cpu(0, cpu_32bit_el0_mask) == cpu_32bit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) if (lucky_winner >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) * We've detected a mismatch. We need to keep one of our CPUs with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) * 32-bit EL0 online so that is_cpu_allowed() doesn't end up rejecting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) * every CPU in the system for a 32-bit task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) cpu_active_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) get_cpu_device(lucky_winner)->offline_disabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) setup_elf_hwcaps(compat_elf_hwcaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) cpu, lucky_winner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) static int __init init_32bit_el0_mask(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) if (!allow_mismatched_32bit_el0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) "arm64/mismatched_32bit_el0:online",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) enable_mismatched_32bit_el0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) subsys_initcall_sync(init_32bit_el0_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) static bool __maybe_unused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) return (__system_matches_cap(ARM64_HAS_PAN) && !__system_matches_cap(ARM64_HAS_UAO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) * We emulate only the following system register space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) * See Table C5-6 System instruction encodings for System register accesses,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) * ARMv8 ARM(ARM DDI 0487A.f) for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) static inline bool __attribute_const__ is_emulated(u32 id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) return (sys_reg_Op0(id) == 0x3 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) sys_reg_CRn(id) == 0x0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) sys_reg_Op1(id) == 0x0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) (sys_reg_CRm(id) == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7))));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) * With CRm == 0, reg should be one of :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) static inline int emulate_id_reg(u32 id, u64 *valp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) switch (id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) case SYS_MIDR_EL1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) *valp = read_cpuid_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) case SYS_MPIDR_EL1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) *valp = SYS_MPIDR_SAFE_VAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) case SYS_REVIDR_EL1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) /* IMPLEMENTATION DEFINED values are emulated with 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) *valp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) static int emulate_sys_reg(u32 id, u64 *valp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) struct arm64_ftr_reg *regp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) if (!is_emulated(id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) if (sys_reg_CRm(id) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) return emulate_id_reg(id, valp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) regp = get_arm64_ftr_reg_nowarn(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) if (regp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) *valp = arm64_ftr_reg_user_value(regp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) * The untracked registers are either IMPLEMENTATION DEFINED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) * (e.g, ID_AFR0_EL1) or reserved RAZ.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) *valp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) rc = emulate_sys_reg(sys_reg, &val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) pt_regs_write_reg(regs, rt, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) static int emulate_mrs(struct pt_regs *regs, u32 insn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) u32 sys_reg, rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) * sys_reg values are defined as used in mrs/msr instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) * shift the imm value to get the encoding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) return do_emulate_mrs(regs, sys_reg, rt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) static struct undef_hook mrs_hook = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) .instr_mask = 0xfff00000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) .instr_val = 0xd5300000,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) .pstate_mask = PSR_AA32_MODE_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) .pstate_val = PSR_MODE_EL0t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) .fn = emulate_mrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) static int __init enable_mrs_emulation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) register_undef_hook(&mrs_hook);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) core_initcall(enable_mrs_emulation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) enum mitigation_state arm64_get_meltdown_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) if (__meltdown_safe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) return SPECTRE_UNAFFECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) if (arm64_kernel_unmapped_at_el0())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) return SPECTRE_MITIGATED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) return SPECTRE_VULNERABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) switch (arm64_get_meltdown_state()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) case SPECTRE_UNAFFECTED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) return sprintf(buf, "Not affected\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) case SPECTRE_MITIGATED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) return sprintf(buf, "Mitigation: PTI\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) return sprintf(buf, "Vulnerable\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags