^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Cyrix stuff, June 1998 by:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * - Rafael R. Reilova (moved everything from head.S),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * <rreilova@ececs.uc.edu>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * - Channing Corn (tests & fixes),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * - Andrew D. Balsa (code cleanup).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/utsname.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/nospec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/prctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/smt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <asm/spec-ctrl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/cmdline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/bugs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/processor-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/fpu/internal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/msr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/vmx.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/paravirt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/intel-family.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/e820/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include "cpu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static void __init spectre_v1_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static void __init spectre_v2_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static void __init ssb_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static void __init l1tf_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static void __init mds_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static void __init mds_print_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static void __init taa_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void __init srbds_select_mitigation(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u64 x86_spec_ctrl_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static DEFINE_MUTEX(spec_ctrl_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * The vendor and possibly platform specific bits which can be modified in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * x86_spec_ctrl_base.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * AMD specific MSR info for Speculative Store Bypass control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) u64 __ro_after_init x86_amd_ls_cfg_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Control conditional STIBP in switch_to() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* Control conditional IBPB in switch_mm() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* Control unconditional IBPB in switch_mm() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* Control MDS CPU buffer clear before returning to user space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) DEFINE_STATIC_KEY_FALSE(mds_user_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) EXPORT_SYMBOL_GPL(mds_user_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* Control MDS CPU buffer clear before idling (halt, mwait) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) EXPORT_SYMBOL_GPL(mds_idle_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) void __init check_bugs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) identify_boot_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * identify_boot_cpu() initialized SMT support information, let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * core code know.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) cpu_smt_check_topology();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!IS_ENABLED(CONFIG_SMP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) pr_info("CPU: ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) print_cpu_info(&boot_cpu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * Read the SPEC_CTRL MSR to account for reserved bits which may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * init code as it is not enumerated and depends on the family.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /* Allow STIBP in MSR_SPEC_CTRL if supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (boot_cpu_has(X86_FEATURE_STIBP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Select the proper CPU mitigations before patching alternatives: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spectre_v1_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) spectre_v2_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ssb_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) l1tf_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) mds_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) taa_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) srbds_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * As MDS and TAA mitigations are inter-related, print MDS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * mitigation until after TAA mitigation selection is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) mds_print_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) arch_smt_update();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Check whether we are able to run this kernel safely on SMP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * - i386 is no longer supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * - In order to run on anything without a TSC, we need to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * compiled for a i486.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (boot_cpu_data.x86 < 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) panic("Kernel requires i486+ for 'invlpg' and other features");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) init_utsname()->machine[1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) alternative_instructions();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) fpu__init_check_bugs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #else /* CONFIG_X86_64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) alternative_instructions();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * Make sure the first 2MB area is not mapped by huge pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * There are typically fixed size MTRRs in there and overlapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * MTRRs into large pages causes slow downs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * Right now we don't do that with gbpages because there seems
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * very little benefit for that case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (!direct_gbpages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) set_memory_4k((unsigned long)__va(0), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) u64 msrval, guestval, hostval = x86_spec_ctrl_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct thread_info *ti = current_thread_info();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /* Is MSR_SPEC_CTRL implemented ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * Restrict guest_spec_ctrl to supported values. Clear the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * modifiable bits in the host base value and or the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * modifiable bits from the guest value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) guestval = hostval & ~x86_spec_ctrl_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* SSBD controlled in MSR_SPEC_CTRL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static_cpu_has(X86_FEATURE_AMD_SSBD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /* Conditional STIBP enabled? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (static_branch_unlikely(&switch_to_cond_stibp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) hostval |= stibp_tif_to_spec_ctrl(ti->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (hostval != guestval) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) msrval = setguest ? guestval : hostval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) !static_cpu_has(X86_FEATURE_VIRT_SSBD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * If the host has SSBD mitigation enabled, force it in the host's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * virtual MSR value. If its not permanently enabled, evaluate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * current's TIF_SSBD thread flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) hostval = SPEC_CTRL_SSBD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) hostval = ssbd_tif_to_spec_ctrl(ti->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /* Sanitize the guest value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (hostval != guestval) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) unsigned long tif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) ssbd_spec_ctrl_to_tif(hostval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) speculation_ctrl_update(tif);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static void x86_amd_ssb_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) wrmsrl(MSR_AMD64_LS_CFG, msrval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) #define pr_fmt(fmt) "MDS: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Default mitigation for MDS-affected CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static bool mds_nosmt __ro_after_init = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static const char * const mds_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) [MDS_MITIGATION_OFF] = "Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) [MDS_MITIGATION_FULL] = "Mitigation: Clear CPU buffers",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) static void __init mds_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) mds_mitigation = MDS_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (mds_mitigation == MDS_MITIGATION_FULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) mds_mitigation = MDS_MITIGATION_VMWERV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static_branch_enable(&mds_user_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) (mds_nosmt || cpu_mitigations_auto_nosmt()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) cpu_smt_disable(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static void __init mds_print_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) pr_info("%s\n", mds_strings[mds_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) static int __init mds_cmdline(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (!boot_cpu_has_bug(X86_BUG_MDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (!str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (!strcmp(str, "off"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) mds_mitigation = MDS_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) else if (!strcmp(str, "full"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) mds_mitigation = MDS_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) else if (!strcmp(str, "full,nosmt")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) mds_mitigation = MDS_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) mds_nosmt = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) early_param("mds", mds_cmdline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #define pr_fmt(fmt) "TAA: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) enum taa_mitigations {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) TAA_MITIGATION_OFF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) TAA_MITIGATION_UCODE_NEEDED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) TAA_MITIGATION_VERW,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) TAA_MITIGATION_TSX_DISABLED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Default mitigation for TAA-affected CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static bool taa_nosmt __ro_after_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static const char * const taa_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) [TAA_MITIGATION_OFF] = "Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) [TAA_MITIGATION_UCODE_NEEDED] = "Vulnerable: Clear CPU buffers attempted, no microcode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) [TAA_MITIGATION_VERW] = "Mitigation: Clear CPU buffers",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) [TAA_MITIGATION_TSX_DISABLED] = "Mitigation: TSX disabled",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static void __init taa_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) u64 ia32_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (!boot_cpu_has_bug(X86_BUG_TAA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) taa_mitigation = TAA_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* TSX previously disabled by tsx=off */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (!boot_cpu_has(X86_FEATURE_RTM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (cpu_mitigations_off()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) taa_mitigation = TAA_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * TAA mitigation via VERW is turned off if both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * tsx_async_abort=off and mds=off are specified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (taa_mitigation == TAA_MITIGATION_OFF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) mds_mitigation == MDS_MITIGATION_OFF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) taa_mitigation = TAA_MITIGATION_VERW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * A microcode update fixes this behavior to clear CPU buffers. It also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * ARCH_CAP_TSX_CTRL_MSR bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * update is required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) ia32_cap = x86_read_arch_cap_msr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * TSX is enabled, select alternate mitigation for TAA which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * the same as MDS. Enable MDS static branch to clear CPU buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * For guests that can't determine whether the correct microcode is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * present on host, enable the mitigation for UCODE_NEEDED as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static_branch_enable(&mds_user_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (taa_nosmt || cpu_mitigations_auto_nosmt())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) cpu_smt_disable(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Update MDS mitigation, if necessary, as the mds_user_clear is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * now enabled for TAA mitigation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (mds_mitigation == MDS_MITIGATION_OFF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) boot_cpu_has_bug(X86_BUG_MDS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) mds_mitigation = MDS_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) mds_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) pr_info("%s\n", taa_strings[taa_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static int __init tsx_async_abort_parse_cmdline(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (!boot_cpu_has_bug(X86_BUG_TAA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (!str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (!strcmp(str, "off")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) taa_mitigation = TAA_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) } else if (!strcmp(str, "full")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) taa_mitigation = TAA_MITIGATION_VERW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) } else if (!strcmp(str, "full,nosmt")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) taa_mitigation = TAA_MITIGATION_VERW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) taa_nosmt = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #define pr_fmt(fmt) "SRBDS: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) enum srbds_mitigations {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) SRBDS_MITIGATION_OFF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) SRBDS_MITIGATION_UCODE_NEEDED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) SRBDS_MITIGATION_FULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) SRBDS_MITIGATION_TSX_OFF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) SRBDS_MITIGATION_HYPERVISOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) static const char * const srbds_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) [SRBDS_MITIGATION_OFF] = "Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) [SRBDS_MITIGATION_FULL] = "Mitigation: Microcode",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) [SRBDS_MITIGATION_TSX_OFF] = "Mitigation: TSX disabled",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) [SRBDS_MITIGATION_HYPERVISOR] = "Unknown: Dependent on hypervisor status",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static bool srbds_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) void update_srbds_msr(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) u64 mcu_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!boot_cpu_has_bug(X86_BUG_SRBDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) switch (srbds_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) case SRBDS_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) case SRBDS_MITIGATION_TSX_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) mcu_ctrl |= RNGDS_MITG_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) case SRBDS_MITIGATION_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) mcu_ctrl &= ~RNGDS_MITG_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static void __init srbds_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) u64 ia32_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (!boot_cpu_has_bug(X86_BUG_SRBDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * Check to see if this is one of the MDS_NO systems supporting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * TSX that are only exposed to SRBDS when TSX is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ia32_cap = x86_read_arch_cap_msr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if ((ia32_cap & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) else if (cpu_mitigations_off() || srbds_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) srbds_mitigation = SRBDS_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) update_srbds_msr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) pr_info("%s\n", srbds_strings[srbds_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static int __init srbds_parse_cmdline(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (!str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (!boot_cpu_has_bug(X86_BUG_SRBDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) srbds_off = !strcmp(str, "off");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) early_param("srbds", srbds_parse_cmdline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) #define pr_fmt(fmt) "Spectre V1 : " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) enum spectre_v1_mitigation {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) SPECTRE_V1_MITIGATION_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) SPECTRE_V1_MITIGATION_AUTO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) SPECTRE_V1_MITIGATION_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) static const char * const spectre_v1_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * Does SMAP provide full mitigation against speculative kernel access to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * userspace?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static bool smap_works_speculatively(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (!boot_cpu_has(X86_FEATURE_SMAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * On CPUs which are vulnerable to Meltdown, SMAP does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * prevent speculative access to user data in the L1 cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * Consider SMAP to be non-functional as a mitigation on these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) static void __init spectre_v1_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * With Spectre v1, a user can speculatively control either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * path of a conditional swapgs with a user-controlled GS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * value. The mitigation is to add lfences to both code paths.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * If FSGSBASE is enabled, the user can put a kernel address in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * GS, in which case SMAP provides no protection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * If FSGSBASE is disabled, the user can only put a user space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * address in GS. That makes an attack harder, but still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * possible if there's no SMAP protection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) !smap_works_speculatively()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * Mitigation can be provided from SWAPGS itself or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * PTI as the CR3 write in the Meltdown mitigation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * is serializing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * If neither is there, mitigate with an LFENCE to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * stop speculation through swapgs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) !boot_cpu_has(X86_FEATURE_PTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * Enable lfences in the kernel entry (non-swapgs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * paths, to prevent user entry from speculatively
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * skipping swapgs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) static int __init nospectre_v1_cmdline(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) early_param("nospectre_v1", nospectre_v1_cmdline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) #define pr_fmt(fmt) "Spectre V2 : " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) SPECTRE_V2_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) SPECTRE_V2_USER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) SPECTRE_V2_USER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) #ifdef CONFIG_RETPOLINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static bool spectre_v2_bad_module;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) bool retpoline_module_ok(bool has_retpoline)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) pr_err("System may be vulnerable to spectre v2\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) spectre_v2_bad_module = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) static inline const char *spectre_v2_module_string(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static inline const char *spectre_v2_module_string(void) { return ""; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) #ifdef CONFIG_BPF_SYSCALL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) void unpriv_ebpf_notify(int new_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (new_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /* Unprivileged eBPF is enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) switch (spectre_v2_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) case SPECTRE_V2_EIBRS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) case SPECTRE_V2_EIBRS_LFENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) static inline bool match_option(const char *arg, int arglen, const char *opt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) int len = strlen(opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) return len == arglen && !strncmp(arg, opt, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* The kernel command line selection for spectre v2 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) enum spectre_v2_mitigation_cmd {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) SPECTRE_V2_CMD_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) SPECTRE_V2_CMD_AUTO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) SPECTRE_V2_CMD_FORCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) SPECTRE_V2_CMD_RETPOLINE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) SPECTRE_V2_CMD_RETPOLINE_GENERIC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) SPECTRE_V2_CMD_RETPOLINE_LFENCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) SPECTRE_V2_CMD_EIBRS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) SPECTRE_V2_CMD_EIBRS_RETPOLINE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) SPECTRE_V2_CMD_EIBRS_LFENCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) enum spectre_v2_user_cmd {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) SPECTRE_V2_USER_CMD_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) SPECTRE_V2_USER_CMD_AUTO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) SPECTRE_V2_USER_CMD_FORCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) SPECTRE_V2_USER_CMD_PRCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) SPECTRE_V2_USER_CMD_PRCTL_IBPB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) SPECTRE_V2_USER_CMD_SECCOMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) static const char * const spectre_v2_user_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) [SPECTRE_V2_USER_NONE] = "User space: Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) [SPECTRE_V2_USER_STRICT_PREFERRED] = "User space: Mitigation: STIBP always-on protection",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) const char *option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) enum spectre_v2_user_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) bool secure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) } v2_user_options[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) { "auto", SPECTRE_V2_USER_CMD_AUTO, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) { "off", SPECTRE_V2_USER_CMD_NONE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) { "on", SPECTRE_V2_USER_CMD_FORCE, true },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) static void __init spec_v2_user_print_cond(const char *reason, bool secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) pr_info("spectre_v2_user=%s forced on command line.\n", reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) static enum spectre_v2_user_cmd __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) char arg[20];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) switch (v2_cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) case SPECTRE_V2_CMD_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return SPECTRE_V2_USER_CMD_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) case SPECTRE_V2_CMD_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return SPECTRE_V2_USER_CMD_FORCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) arg, sizeof(arg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return SPECTRE_V2_USER_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (match_option(arg, ret, v2_user_options[i].option)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) spec_v2_user_print_cond(v2_user_options[i].option,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) v2_user_options[i].secure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) return v2_user_options[i].cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return SPECTRE_V2_USER_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return (mode == SPECTRE_V2_EIBRS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) mode == SPECTRE_V2_EIBRS_RETPOLINE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) mode == SPECTRE_V2_EIBRS_LFENCE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) static void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) bool smt_possible = IS_ENABLED(CONFIG_SMP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) enum spectre_v2_user_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) smt_possible = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) cmd = spectre_v2_parse_user_cmdline(v2_cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) case SPECTRE_V2_USER_CMD_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) goto set_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) case SPECTRE_V2_USER_CMD_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) mode = SPECTRE_V2_USER_STRICT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) case SPECTRE_V2_USER_CMD_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) mode = SPECTRE_V2_USER_PRCTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) case SPECTRE_V2_USER_CMD_AUTO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) case SPECTRE_V2_USER_CMD_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (IS_ENABLED(CONFIG_SECCOMP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) mode = SPECTRE_V2_USER_SECCOMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) mode = SPECTRE_V2_USER_PRCTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) /* Initialize Indirect Branch Prediction Barrier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (boot_cpu_has(X86_FEATURE_IBPB)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) spectre_v2_user_ibpb = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) case SPECTRE_V2_USER_CMD_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) static_branch_enable(&switch_mm_always_ibpb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) case SPECTRE_V2_USER_CMD_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) case SPECTRE_V2_USER_CMD_AUTO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) case SPECTRE_V2_USER_CMD_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) static_branch_enable(&switch_mm_cond_ibpb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) static_key_enabled(&switch_mm_always_ibpb) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) "always-on" : "conditional");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * If no STIBP, enhanced IBRS is enabled or SMT impossible, STIBP is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) if (!boot_cpu_has(X86_FEATURE_STIBP) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) !smt_possible ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) spectre_v2_in_eibrs_mode(spectre_v2_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * At this point, an STIBP mode other than "off" has been set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * If STIBP support is not being forced, check if STIBP always-on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * is preferred.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (mode != SPECTRE_V2_USER_STRICT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) mode = SPECTRE_V2_USER_STRICT_PREFERRED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) spectre_v2_user_stibp = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) set_mode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) pr_info("%s\n", spectre_v2_user_strings[mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) static const char * const spectre_v2_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) [SPECTRE_V2_NONE] = "Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) [SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) [SPECTRE_V2_LFENCE] = "Mitigation: LFENCE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) [SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) [SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) [SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) const char *option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) enum spectre_v2_mitigation_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) bool secure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) } mitigation_options[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) { "off", SPECTRE_V2_CMD_NONE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) { "on", SPECTRE_V2_CMD_FORCE, true },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) { "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) { "eibrs", SPECTRE_V2_CMD_EIBRS, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) { "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) { "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) { "auto", SPECTRE_V2_CMD_AUTO, false },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) static void __init spec_v2_print_cond(const char *reason, bool secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) pr_info("%s selected on command line.\n", reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) char arg[20];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) cpu_mitigations_off())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) return SPECTRE_V2_CMD_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) return SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) if (!match_option(arg, ret, mitigation_options[i].option))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) cmd = mitigation_options[i].cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (i >= ARRAY_SIZE(mitigation_options)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) pr_err("unknown option (%s). Switching to AUTO select\n", arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) return SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) !IS_ENABLED(CONFIG_RETPOLINE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) pr_err("%s selected but not compiled in. Switching to AUTO select\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) mitigation_options[i].option);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) return SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if ((cmd == SPECTRE_V2_CMD_EIBRS ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) mitigation_options[i].option);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) return SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) mitigation_options[i].option);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) return SPECTRE_V2_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) spec_v2_print_cond(mitigation_options[i].option,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) mitigation_options[i].secure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) return cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) if (!IS_ENABLED(CONFIG_RETPOLINE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) pr_err("Kernel not compiled with retpoline; no mitigation available!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) return SPECTRE_V2_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return SPECTRE_V2_RETPOLINE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) static void __init spectre_v2_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * If the CPU is not affected and the command line mode is NONE or AUTO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * then nothing to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) case SPECTRE_V2_CMD_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) case SPECTRE_V2_CMD_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) case SPECTRE_V2_CMD_AUTO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) mode = SPECTRE_V2_EIBRS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) mode = spectre_v2_select_retpoline();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) pr_err(SPECTRE_V2_LFENCE_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) mode = SPECTRE_V2_LFENCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) mode = SPECTRE_V2_RETPOLINE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) case SPECTRE_V2_CMD_RETPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) mode = spectre_v2_select_retpoline();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) case SPECTRE_V2_CMD_EIBRS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) mode = SPECTRE_V2_EIBRS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) case SPECTRE_V2_CMD_EIBRS_LFENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) mode = SPECTRE_V2_EIBRS_LFENCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) mode = SPECTRE_V2_EIBRS_RETPOLINE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (spectre_v2_in_eibrs_mode(mode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) /* Force it so VMEXIT will restore correctly */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) case SPECTRE_V2_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) case SPECTRE_V2_EIBRS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) case SPECTRE_V2_LFENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) case SPECTRE_V2_EIBRS_LFENCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) case SPECTRE_V2_RETPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) case SPECTRE_V2_EIBRS_RETPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) spectre_v2_enabled = mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) pr_info("%s\n", spectre_v2_strings[mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) * If spectre v2 protection has been enabled, unconditionally fill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) * RSB during a context switch; this protects against two independent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * issues:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * - RSB underflow (and switch to BTB) on Skylake+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * Retpoline means the kernel is safe because it has no indirect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * branches. Enhanced IBRS protects firmware too, so, enable restricted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) * speculation around firmware calls only when Enhanced IBRS isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) * supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) * the user might select retpoline on the kernel command line and if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) * the CPU supports Enhanced IBRS, kernel might un-intentionally not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) * enable IBRS around firmware calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) pr_info("Enabling Restricted Speculation for firmware calls\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) /* Set up IBPB and STIBP depending on the general spectre V2 command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) spectre_v2_user_select_mitigation(cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) static void update_stibp_msr(void * __unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) /* Update x86_spec_ctrl_base in case SMT state changed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) static void update_stibp_strict(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) mask |= SPEC_CTRL_STIBP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (mask == x86_spec_ctrl_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) pr_info("Update user space SMT mitigation: STIBP %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) mask & SPEC_CTRL_STIBP ? "always-on" : "off");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) x86_spec_ctrl_base = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) on_each_cpu(update_stibp_msr, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) /* Update the static key controlling the evaluation of TIF_SPEC_IB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static void update_indir_branch_cond(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) static_branch_enable(&switch_to_cond_stibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) static_branch_disable(&switch_to_cond_stibp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) #define pr_fmt(fmt) fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /* Update the static key controlling the MDS CPU buffer clear in idle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) static void update_mds_branch_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * Enable the idle clearing if SMT is active on CPUs which are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * affected only by MSBDS and not any other MDS variant.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * The other variants cannot be mitigated when SMT is enabled, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * clearing the buffers on idle just to prevent the Store Buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) * repartitioning leak would be a window dressing exercise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) static_branch_enable(&mds_idle_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) static_branch_disable(&mds_idle_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) void cpu_bugs_smt_update(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) mutex_lock(&spec_ctrl_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) if (sched_smt_active() && unprivileged_ebpf_enabled() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) switch (spectre_v2_user_stibp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) case SPECTRE_V2_USER_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) case SPECTRE_V2_USER_STRICT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) case SPECTRE_V2_USER_STRICT_PREFERRED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) update_stibp_strict();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) case SPECTRE_V2_USER_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) case SPECTRE_V2_USER_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) update_indir_branch_cond();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) switch (mds_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) case MDS_MITIGATION_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) case MDS_MITIGATION_VMWERV:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) pr_warn_once(MDS_MSG_SMT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) update_mds_branch_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) case MDS_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) switch (taa_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) case TAA_MITIGATION_VERW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) case TAA_MITIGATION_UCODE_NEEDED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (sched_smt_active())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) pr_warn_once(TAA_MSG_SMT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) case TAA_MITIGATION_TSX_DISABLED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) case TAA_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) mutex_unlock(&spec_ctrl_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) #define pr_fmt(fmt) "Speculative Store Bypass: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) /* The kernel command line selection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) enum ssb_mitigation_cmd {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) SPEC_STORE_BYPASS_CMD_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) SPEC_STORE_BYPASS_CMD_AUTO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) SPEC_STORE_BYPASS_CMD_ON,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) SPEC_STORE_BYPASS_CMD_PRCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) SPEC_STORE_BYPASS_CMD_SECCOMP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) static const char * const ssb_strings[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) [SPEC_STORE_BYPASS_NONE] = "Vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) [SPEC_STORE_BYPASS_SECCOMP] = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) const char *option;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) enum ssb_mitigation_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) } ssb_mitigation_options[] __initconst = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) { "prctl", SPEC_STORE_BYPASS_CMD_PRCTL }, /* Disable Speculative Store Bypass via prctl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) { "seccomp", SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) char arg[20];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) int ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) cpu_mitigations_off()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) return SPEC_STORE_BYPASS_CMD_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) arg, sizeof(arg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) return SPEC_STORE_BYPASS_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (!match_option(arg, ret, ssb_mitigation_options[i].option))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) cmd = ssb_mitigation_options[i].cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) pr_err("unknown option (%s). Switching to AUTO select\n", arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return SPEC_STORE_BYPASS_CMD_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) return cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) static enum ssb_mitigation __init __ssb_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) enum ssb_mitigation_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) if (!boot_cpu_has(X86_FEATURE_SSBD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) cmd = ssb_parse_cmdline();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) cmd == SPEC_STORE_BYPASS_CMD_AUTO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) case SPEC_STORE_BYPASS_CMD_AUTO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) case SPEC_STORE_BYPASS_CMD_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * Choose prctl+seccomp as the default mode if seccomp is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (IS_ENABLED(CONFIG_SECCOMP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) mode = SPEC_STORE_BYPASS_SECCOMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) mode = SPEC_STORE_BYPASS_PRCTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) case SPEC_STORE_BYPASS_CMD_ON:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) mode = SPEC_STORE_BYPASS_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) case SPEC_STORE_BYPASS_CMD_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) mode = SPEC_STORE_BYPASS_PRCTL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) case SPEC_STORE_BYPASS_CMD_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * bit in the mask to allow guests to use the mitigation even in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * case where the host does not enable it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) static_cpu_has(X86_FEATURE_AMD_SSBD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * We have three CPU feature flags that are in play here:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) * - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) * - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) if (mode == SPEC_STORE_BYPASS_DISABLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * use a completely different MSR and bit dependent on family.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) x86_amd_ssb_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) static void ssb_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) ssb_mode = __ssb_select_mitigation();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) pr_info("%s\n", ssb_strings[ssb_mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) #define pr_fmt(fmt) "Speculation prctl: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) static void task_update_spec_tif(struct task_struct *tsk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) /* Force the update of the real TIF bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) * Immediately update the speculation control MSRs for the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) * task, but for a non-current task delay setting the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) * mitigation until it is scheduled next.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * This can only happen for SECCOMP mitigation. For PRCTL it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * always the current task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) if (tsk == current)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) speculation_ctrl_update_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) switch (ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) case PR_SPEC_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) /* If speculation is force disabled, enable is not allowed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) if (task_spec_ssb_force_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) task_clear_spec_ssb_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) task_clear_spec_ssb_noexec(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) case PR_SPEC_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) task_set_spec_ssb_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) task_clear_spec_ssb_noexec(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) case PR_SPEC_FORCE_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) task_set_spec_ssb_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) task_set_spec_ssb_force_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) task_clear_spec_ssb_noexec(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) case PR_SPEC_DISABLE_NOEXEC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) if (task_spec_ssb_force_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) task_set_spec_ssb_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) task_set_spec_ssb_noexec(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) static bool is_spec_ib_user_controlled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) switch (ctrl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) case PR_SPEC_ENABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) * With strict mode for both IBPB and STIBP, the instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) * code paths avoid checking this task flag and instead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) * unconditionally run the instruction. However, STIBP and IBPB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * are independent and either can be set to conditionally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * enabled regardless of the mode of the other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * If either is set to conditional, allow the task flag to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * updated, unless it was force-disabled by a previous prctl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * call. Currently, this is possible on an AMD CPU which has the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * kernel is booted with 'spectre_v2_user=seccomp', then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) if (!is_spec_ib_user_controlled() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) task_spec_ib_force_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) task_clear_spec_ib_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) case PR_SPEC_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) case PR_SPEC_FORCE_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * Indirect branch speculation is always allowed when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * mitigation is force disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) if (!is_spec_ib_user_controlled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) task_set_spec_ib_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) if (ctrl == PR_SPEC_FORCE_DISABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) task_set_spec_ib_force_disable(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) task_update_spec_tif(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) unsigned long ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) switch (which) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) case PR_SPEC_STORE_BYPASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) return ssb_prctl_set(task, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) case PR_SPEC_INDIRECT_BRANCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) return ib_prctl_set(task, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) #ifdef CONFIG_SECCOMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) void arch_seccomp_spec_mitigate(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) static int ssb_prctl_get(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) switch (ssb_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) case SPEC_STORE_BYPASS_DISABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) return PR_SPEC_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) case SPEC_STORE_BYPASS_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) case SPEC_STORE_BYPASS_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (task_spec_ssb_force_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (task_spec_ssb_noexec(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) if (task_spec_ssb_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) return PR_SPEC_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return PR_SPEC_NOT_AFFECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) static int ib_prctl_get(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) return PR_SPEC_NOT_AFFECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) return PR_SPEC_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) else if (is_spec_ib_user_controlled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) if (task_spec_ib_force_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) if (task_spec_ib_disable(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) return PR_SPEC_DISABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) return PR_SPEC_NOT_AFFECTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) switch (which) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) case PR_SPEC_STORE_BYPASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return ssb_prctl_get(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) case PR_SPEC_INDIRECT_BRANCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) return ib_prctl_get(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) void x86_spec_ctrl_setup_ap(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) x86_amd_ssb_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) bool itlb_multihit_kvm_mitigation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) #define pr_fmt(fmt) "L1TF: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) /* Default mitigation for L1TF-affected CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) #if IS_ENABLED(CONFIG_KVM_INTEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) EXPORT_SYMBOL_GPL(l1tf_mitigation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) * These CPUs all support 44bits physical address space internally in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) * cache but CPUID can report a smaller number of physical address bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) * The L1TF mitigation uses the top most address bit for the inversion of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) * non present PTEs. When the installed memory reaches into the top most
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) * address bit due to memory holes, which has been observed on machines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) * which report 36bits physical address bits and have 32G RAM installed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) * then the mitigation range check in l1tf_select_mitigation() triggers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) * This is a false positive because the mitigation is still possible due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) * the fact that the cache uses 44bit internally. Use the cache bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) * instead of the reported physical bits and adjust them on the affected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) * machines to 44bit if the reported bits are less than 44.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) static void override_cache_bits(struct cpuinfo_x86 *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) if (c->x86 != 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) switch (c->x86_model) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) case INTEL_FAM6_NEHALEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) case INTEL_FAM6_WESTMERE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) case INTEL_FAM6_SANDYBRIDGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) case INTEL_FAM6_IVYBRIDGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) case INTEL_FAM6_HASWELL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) case INTEL_FAM6_HASWELL_L:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) case INTEL_FAM6_HASWELL_G:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) case INTEL_FAM6_BROADWELL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) case INTEL_FAM6_BROADWELL_G:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) case INTEL_FAM6_SKYLAKE_L:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) case INTEL_FAM6_SKYLAKE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) case INTEL_FAM6_KABYLAKE_L:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) case INTEL_FAM6_KABYLAKE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (c->x86_cache_bits < 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) c->x86_cache_bits = 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) static void __init l1tf_select_mitigation(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) u64 half_pa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) if (!boot_cpu_has_bug(X86_BUG_L1TF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) if (cpu_mitigations_off())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) l1tf_mitigation = L1TF_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) else if (cpu_mitigations_auto_nosmt())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) override_cache_bits(&boot_cpu_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) switch (l1tf_mitigation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) case L1TF_MITIGATION_OFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) case L1TF_MITIGATION_FLUSH_NOWARN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) case L1TF_MITIGATION_FLUSH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) case L1TF_MITIGATION_FLUSH_NOSMT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) case L1TF_MITIGATION_FULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) cpu_smt_disable(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) case L1TF_MITIGATION_FULL_FORCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) cpu_smt_disable(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) #if CONFIG_PGTABLE_LEVELS == 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) half_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) pr_info("However, doing so will make a part of your RAM unusable.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) static int __init l1tf_cmdline(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) if (!boot_cpu_has_bug(X86_BUG_L1TF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) if (!str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) if (!strcmp(str, "off"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) l1tf_mitigation = L1TF_MITIGATION_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) else if (!strcmp(str, "flush,nowarn"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) else if (!strcmp(str, "flush"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) l1tf_mitigation = L1TF_MITIGATION_FLUSH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) else if (!strcmp(str, "flush,nosmt"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) else if (!strcmp(str, "full"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) l1tf_mitigation = L1TF_MITIGATION_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) else if (!strcmp(str, "full,force"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) early_param("l1tf", l1tf_cmdline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) #undef pr_fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) #define pr_fmt(fmt) fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) #ifdef CONFIG_SYSFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) #if IS_ENABLED(CONFIG_KVM_INTEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) static const char * const l1tf_vmx_states[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) [VMENTER_L1D_FLUSH_AUTO] = "auto",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) [VMENTER_L1D_FLUSH_NEVER] = "vulnerable",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) [VMENTER_L1D_FLUSH_ALWAYS] = "cache flushes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) [VMENTER_L1D_FLUSH_EPT_DISABLED] = "EPT disabled",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) [VMENTER_L1D_FLUSH_NOT_REQUIRED] = "flush not necessary"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) static ssize_t l1tf_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) sched_smt_active())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) l1tf_vmx_states[l1tf_vmx_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) l1tf_vmx_states[l1tf_vmx_mitigation],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) sched_smt_active() ? "vulnerable" : "disabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) static ssize_t itlb_multihit_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) !boot_cpu_has(X86_FEATURE_VMX))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) return sprintf(buf, "KVM: Mitigation: VMX unsupported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) else if (!(cr4_read_shadow() & X86_CR4_VMXE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) return sprintf(buf, "KVM: Mitigation: VMX disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) else if (itlb_multihit_kvm_mitigation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) return sprintf(buf, "KVM: Vulnerable\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) static ssize_t l1tf_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) static ssize_t itlb_multihit_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) return sprintf(buf, "Processor vulnerable\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) static ssize_t mds_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) return sprintf(buf, "%s; SMT Host state unknown\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) mds_strings[mds_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) sched_smt_active() ? "mitigated" : "disabled"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) sched_smt_active() ? "vulnerable" : "disabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) static ssize_t tsx_async_abort_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) (taa_mitigation == TAA_MITIGATION_OFF))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) return sprintf(buf, "%s; SMT Host state unknown\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) taa_strings[taa_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) sched_smt_active() ? "vulnerable" : "disabled");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) static char *stibp_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) return "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) switch (spectre_v2_user_stibp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) case SPECTRE_V2_USER_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) return ", STIBP: disabled";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) case SPECTRE_V2_USER_STRICT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) return ", STIBP: forced";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) case SPECTRE_V2_USER_STRICT_PREFERRED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) return ", STIBP: always-on";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) case SPECTRE_V2_USER_PRCTL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) case SPECTRE_V2_USER_SECCOMP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) if (static_key_enabled(&switch_to_cond_stibp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) return ", STIBP: conditional";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) return "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) static char *ibpb_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) if (boot_cpu_has(X86_FEATURE_IBPB)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) if (static_key_enabled(&switch_mm_always_ibpb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) return ", IBPB: always-on";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (static_key_enabled(&switch_mm_cond_ibpb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) return ", IBPB: conditional";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) return ", IBPB: disabled";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) return "";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) static ssize_t spectre_v2_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) return sprintf(buf, "Vulnerable: LFENCE\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) return sprintf(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) if (sched_smt_active() && unprivileged_ebpf_enabled() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) return sprintf(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) return sprintf(buf, "%s%s%s%s%s%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) spectre_v2_strings[spectre_v2_enabled],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) ibpb_state(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) stibp_state(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) spectre_v2_module_string());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) static ssize_t srbds_show_state(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) return sprintf(buf, "%s\n", srbds_strings[srbds_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) char *buf, unsigned int bug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) if (!boot_cpu_has_bug(bug))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) return sprintf(buf, "Not affected\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) switch (bug) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) case X86_BUG_CPU_MELTDOWN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) if (boot_cpu_has(X86_FEATURE_PTI))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) return sprintf(buf, "Mitigation: PTI\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) if (hypervisor_is_type(X86_HYPER_XEN_PV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) case X86_BUG_SPECTRE_V1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) case X86_BUG_SPECTRE_V2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) return spectre_v2_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) case X86_BUG_SPEC_STORE_BYPASS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) case X86_BUG_L1TF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) return l1tf_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) case X86_BUG_MDS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) return mds_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) case X86_BUG_TAA:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) return tsx_async_abort_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) case X86_BUG_ITLB_MULTIHIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) return itlb_multihit_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) case X86_BUG_SRBDS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) return srbds_show_state(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) return sprintf(buf, "Vulnerable\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags