^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _ASM_X86_MSR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _ASM_X86_MSR_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include "msr-index.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/asm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <uapi/asm/msr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) struct msr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) u32 l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) u32 h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) u64 q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct msr_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) u32 msr_no;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) struct msr reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) struct msr *msrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) struct msr_regs_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) u32 *regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) struct saved_msr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) bool valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct msr_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct saved_msrs {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) unsigned int num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct saved_msr *array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * constraint has different meanings. For i386, "A" means exactly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * it means rax *or* rdx.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* Using 64-bit values saves one instruction clearing the high half of low */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define DECLARE_ARGS(val, low, high) unsigned long low, high
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define DECLARE_ARGS(val, low, high) unsigned long long val
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define EAX_EDX_VAL(val, low, high) (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define EAX_EDX_RET(val, low, high) "=A" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * Be very careful with includes. This header is prone to include loops.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <asm/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <linux/tracepoint-defs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #ifdef CONFIG_TRACEPOINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) DECLARE_TRACEPOINT(read_msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) DECLARE_TRACEPOINT(write_msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) DECLARE_TRACEPOINT(rdpmc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * accessors and should not have any tracing or other functionality piggybacking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * on them - those are *purely* for accessing MSRs and nothing more. So don't even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * think of extending them - you will be slapped with a stinking trout or a frozen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * shark will reach you, wherever you are! You've been warned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static __always_inline unsigned long long __rdmsr(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) DECLARE_ARGS(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) asm volatile("1: rdmsr\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) "2:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_unsafe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) : EAX_EDX_RET(val, low, high) : "c" (msr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return EAX_EDX_VAL(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) asm volatile("1: wrmsr\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) "2:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_unsafe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) : : "c" (msr), "a"(low), "d" (high) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define native_rdmsr(msr, val1, val2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) u64 __val = __rdmsr((msr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) (void)((val1) = (u32)__val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) (void)((val2) = (u32)(__val >> 32)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define native_wrmsr(msr, low, high) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) __wrmsr(msr, low, high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define native_wrmsrl(msr, val) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __wrmsr((msr), (u32)((u64)(val)), \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) (u32)((u64)(val) >> 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static inline unsigned long long native_read_msr(unsigned int msr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned long long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) val = __rdmsr(msr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (tracepoint_enabled(read_msr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) do_trace_read_msr(msr, val, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static inline unsigned long long native_read_msr_safe(unsigned int msr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) int *err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) DECLARE_ARGS(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) asm volatile("2: rdmsr ; xor %[err],%[err]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) "1:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) ".section .fixup,\"ax\"\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) "3: mov %[fault],%[err]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) "xorl %%eax, %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) "xorl %%edx, %%edx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) "jmp 1b\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) ".previous\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) _ASM_EXTABLE(2b, 3b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) : "c" (msr), [fault] "i" (-EIO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) if (tracepoint_enabled(read_msr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return EAX_EDX_VAL(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /* Can be uninlined because referenced by paravirt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static inline void notrace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) native_write_msr(unsigned int msr, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) __wrmsr(msr, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (tracepoint_enabled(write_msr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /* Can be uninlined because referenced by paravirt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline int notrace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) native_write_msr_safe(unsigned int msr, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) asm volatile("2: wrmsr ; xor %[err],%[err]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) "1:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) ".section .fixup,\"ax\"\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) "3: mov %[fault],%[err] ; jmp 1b\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) ".previous\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) _ASM_EXTABLE(2b, 3b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) : [err] "=a" (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) : "c" (msr), "0" (low), "d" (high),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) [fault] "i" (-EIO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (tracepoint_enabled(write_msr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) do_trace_write_msr(msr, ((u64)high << 32 | low), err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) extern int rdmsr_safe_regs(u32 regs[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) extern int wrmsr_safe_regs(u32 regs[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * rdtsc() - returns the current TSC without ordering constraints
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * rdtsc() returns the result of RDTSC as a 64-bit integer. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * only ordering constraint it supplies is the ordering implied by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * "asm volatile": it will put the RDTSC in the place you expect. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * CPU can and will speculatively execute that RDTSC, though, so the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * results can be non-monotonic if compared on different CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static __always_inline unsigned long long rdtsc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) DECLARE_ARGS(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return EAX_EDX_VAL(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * rdtsc_ordered() - read the current TSC in program order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * It is ordered like a load to a global in-memory counter. It should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * be impossible to observe non-monotonic rdtsc_unordered() behavior
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * across multiple CPUs as long as the TSC is synced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static __always_inline unsigned long long rdtsc_ordered(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) DECLARE_ARGS(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * The RDTSC instruction is not ordered relative to memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * access. The Intel SDM and the AMD APM are both vague on this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * point, but empirically an RDTSC instruction can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * speculatively executed before prior loads. An RDTSC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * immediately after an appropriate barrier appears to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * ordered as a normal load, that is, it provides the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * ordering guarantees as reading from a global memory location
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * that some other imaginary CPU is updating continuously with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * time stamp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * Thus, use the preferred barrier on the respective CPU, aiming for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * RDTSCP as the default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) asm volatile(ALTERNATIVE_2("rdtsc",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) "rdtscp", X86_FEATURE_RDTSCP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) : EAX_EDX_RET(val, low, high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) :: "ecx");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return EAX_EDX_VAL(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static inline unsigned long long native_read_pmc(int counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) DECLARE_ARGS(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (tracepoint_enabled(rdpmc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return EAX_EDX_VAL(val, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #ifdef CONFIG_PARAVIRT_XXL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) #include <asm/paravirt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * Access to machine-specific registers (available on 586 and better only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * Note: the rd* operations modify the parameters directly (without using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * pointer indirection), this allows gcc to optimize better
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #define rdmsr(msr, low, high) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) u64 __val = native_read_msr((msr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) (void)((low) = (u32)__val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) (void)((high) = (u32)(__val >> 32)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static inline void wrmsr(unsigned int msr, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) native_write_msr(msr, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #define rdmsrl(msr, val) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) ((val) = native_read_msr((msr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static inline void wrmsrl(unsigned int msr, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* wrmsr with exception handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return native_write_msr_safe(msr, low, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /* rdmsr with exception handling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #define rdmsr_safe(msr, low, high) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) int __err; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) u64 __val = native_read_msr_safe((msr), &__err); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) (*low) = (u32)__val; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) (*high) = (u32)(__val >> 32); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) __err; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) *p = native_read_msr_safe(msr, &err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #define rdpmc(counter, low, high) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) u64 _l = native_read_pmc((counter)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) (low) = (u32)_l; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) (high) = (u32)(_l >> 32); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) #define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #endif /* !CONFIG_PARAVIRT_XXL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * 64-bit version of wrmsr_safe():
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static inline int wrmsrl_safe(u32 msr, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return wrmsr_safe(msr, (u32)val, (u32)(val >> 32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) #define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) struct msr *msrs_alloc(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) void msrs_free(struct msr *msrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) int msr_set_bit(u32 msr, u8 bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) int msr_clear_bit(u32 msr, u8 bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #else /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) rdmsr(msr_no, *l, *h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) wrmsr(msr_no, l, h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) rdmsrl(msr_no, *q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) wrmsrl(msr_no, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct msr *msrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct msr *msrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) u32 *l, u32 *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return rdmsr_safe(msr_no, l, h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return wrmsr_safe(msr_no, l, h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return rdmsrl_safe(msr_no, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return wrmsrl_safe(msr_no, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return rdmsr_safe_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return wrmsr_safe_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) #endif /* __ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) #endif /* _ASM_X86_MSR_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags