^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef __ASM_LSE_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define __ASM_LSE_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <asm/atomic_ll_sc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #if defined(CONFIG_ARM64_LSE_ATOMICS) && !defined(BUILD_FIPS140_KO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #define __LSE_PREAMBLE ".arch_extension lse\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/compiler_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/stringify.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/atomic_lse.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/cpucaps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) extern struct static_key_false arm64_const_caps_ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static inline bool system_uses_lse_atomics(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) return (static_branch_likely(&arm64_const_caps_ready)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define __lse_ll_sc_body(op, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) system_uses_lse_atomics() ? \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) __lse_##op(__VA_ARGS__) : \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) __ll_sc_##op(__VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) /* In-line patching at runtime */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define ARM64_LSE_ATOMIC_INSN(llsc, lse) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) ALTERNATIVE(llsc, __LSE_PREAMBLE lse, ARM64_HAS_LSE_ATOMICS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #else /* CONFIG_ARM64_LSE_ATOMICS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static inline bool system_uses_lse_atomics(void) { return false; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define __lse_ll_sc_body(op, ...) __ll_sc_##op(__VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define ARM64_LSE_ATOMIC_INSN(llsc, lse) llsc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #endif /* CONFIG_ARM64_LSE_ATOMICS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #endif /* __ASM_LSE_H */