^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <asm/nospec-branch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) static int __init nobp_setup_early(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) bool enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) rc = kstrtobool(str, &enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) if (enabled && test_facility(82)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * The user explicitely requested nobp=1, enable it and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * disable the expoline support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) __set_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) if (IS_ENABLED(CONFIG_EXPOLINE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) nospec_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) early_param("nobp", nobp_setup_early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static int __init nospec_setup_early(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) early_param("nospec", nospec_setup_early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static int __init nospec_report(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (test_facility(156))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) pr_info("Spectre V2 mitigation: etokens\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) pr_info("Spectre V2 mitigation: execute trampolines\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (__test_facility(82, S390_lowcore.alt_stfle_fac_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) pr_info("Spectre V2 mitigation: limited branch prediction\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) arch_initcall(nospec_report);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #ifdef CONFIG_EXPOLINE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) int nospec_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static int __init nospectre_v2_setup_early(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) nospec_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) early_param("nospectre_v2", nospectre_v2_setup_early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) void __init nospec_auto_detect(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) if (test_facility(156) || cpu_mitigations_off()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * The machine supports etokens.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * Disable expolines and disable nobp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if (__is_defined(CC_USING_EXPOLINE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) nospec_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) } else if (__is_defined(CC_USING_EXPOLINE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * The kernel has been compiled with expolines.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Keep expolines enabled and disable nobp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) nospec_disable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * If the kernel has not been compiled with expolines the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * nobp setting decides what is done, this depends on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * CONFIG_KERNEL_NP option and the nobp/nospec parameters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static int __init spectre_v2_setup_early(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (str && !strncmp(str, "on", 2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) nospec_disable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (str && !strncmp(str, "off", 3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) nospec_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (str && !strncmp(str, "auto", 4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) nospec_auto_detect();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) early_param("spectre_v2", spectre_v2_setup_early);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static void __init_or_module __nospec_revert(s32 *start, s32 *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) u8 *instr, *thunk, *br;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) u8 insnbuf[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) s32 *epo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /* Second part of the instruction replace is always a nop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) for (epo = start; epo < end; epo++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) instr = (u8 *) epo + *epo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) type = BRCL_EXPOLINE; /* brcl instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) type = BRASL_EXPOLINE; /* brasl instruction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) thunk = instr + (*(int *)(instr + 2)) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) if (thunk[0] == 0xc6 && thunk[1] == 0x00)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /* exrl %r0,<target-br> */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) br = thunk + (*(int *)(thunk + 2)) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) thunk[6] == 0x44 && thunk[7] == 0x00 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) (thunk[1] & 0xf0) == (thunk[8] & 0xf0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* larl %rx,<target br> + ex %r0,0(%rx) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) br = thunk + (*(int *)(thunk + 2)) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /* Check for unconditional branch 0x07f? or 0x47f???? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if ((br[0] & 0xbf) != 0x07 || (br[1] & 0xf0) != 0xf0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x07, 0x00 }, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) case BRCL_EXPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) insnbuf[0] = br[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (br[0] == 0x47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* brcl to b, replace with bc + nopr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) insnbuf[2] = br[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) insnbuf[3] = br[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /* brcl to br, replace with bcr + nop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) case BRASL_EXPOLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (br[0] == 0x47) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /* brasl to b, replace with bas + nopr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) insnbuf[0] = 0x4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) insnbuf[2] = br[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) insnbuf[3] = br[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* brasl to br, replace with basr + nop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) insnbuf[0] = 0x0d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) s390_kernel_write(instr, insnbuf, 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) void __init_or_module nospec_revert(s32 *start, s32 *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (nospec_disable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) __nospec_revert(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) extern s32 __nospec_call_start[], __nospec_call_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) extern s32 __nospec_return_start[], __nospec_return_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void __init nospec_init_branches(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) nospec_revert(__nospec_call_start, __nospec_call_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) nospec_revert(__nospec_return_start, __nospec_return_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #endif /* CONFIG_EXPOLINE */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags