^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef __ARCH_S390_PERCPU__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define __ARCH_S390_PERCPU__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/cmpxchg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * s390 uses its own implementation for per cpu data, the offset of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * the cpu local data area is cached in the cpu's lowcore memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define __my_cpu_offset S390_lowcore.percpu_offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * For 64 bit module code, the module may be more than 4G above the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * per cpu area, use weak definitions to force the compiler to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * generate external references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #if defined(MODULE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define ARCH_NEEDS_WEAK_PER_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * We use a compare-and-swap loop since that uses less cpu cycles than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * disabling and enabling interrupts like the generic variant would do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define arch_this_cpu_to_op_simple(pcp, val, op) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) typedef typeof(pcp) pcp_op_T__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) pcp_op_T__ old__, new__, prev__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) pcp_op_T__ *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) prev__ = *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) old__ = prev__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) new__ = old__ op (val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) prev__ = cmpxchg(ptr__, old__, new__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) } while (prev__ != old__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) new__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #ifndef CONFIG_HAVE_MARCH_Z196_FEATURES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) typedef typeof(pcp) pcp_op_T__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) pcp_op_T__ val__ = (val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) pcp_op_T__ old__, *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (__builtin_constant_p(val__) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) asm volatile( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) op2 " %[ptr__],%[val__]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) : [ptr__] "+Q" (*ptr__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) : [val__] "i" ((szcast)val__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) : "cc"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) } else { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) asm volatile( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) op1 " %[old__],%[val__],%[ptr__]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) : [val__] "d" (val__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) : "cc"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define arch_this_cpu_add_return(pcp, val, op) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) typedef typeof(pcp) pcp_op_T__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) pcp_op_T__ val__ = (val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) pcp_op_T__ old__, *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) asm volatile( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) op " %[old__],%[val__],%[ptr__]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) : [val__] "d" (val__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) : "cc"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) old__ + val__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define arch_this_cpu_to_op(pcp, val, op) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) typedef typeof(pcp) pcp_op_T__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) pcp_op_T__ val__ = (val); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) pcp_op_T__ old__, *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) asm volatile( \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) op " %[old__],%[val__],%[ptr__]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) : [val__] "d" (val__) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) : "cc"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define arch_this_cpu_cmpxchg(pcp, oval, nval) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) typedef typeof(pcp) pcp_op_T__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) pcp_op_T__ ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) pcp_op_T__ *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) ret__ = cmpxchg(ptr__, oval, nval); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #define arch_this_cpu_xchg(pcp, nval) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) typeof(pcp) *ptr__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) typeof(pcp) ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) ptr__ = raw_cpu_ptr(&(pcp)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) ret__ = xchg(ptr__, nval); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define arch_this_cpu_cmpxchg_double(pcp1, pcp2, o1, o2, n1, n2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) typeof(pcp1) o1__ = (o1), n1__ = (n1); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) typeof(pcp2) o2__ = (o2), n2__ = (n2); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) typeof(pcp1) *p1__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) typeof(pcp2) *p2__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) preempt_disable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) p1__ = raw_cpu_ptr(&(pcp1)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) p2__ = raw_cpu_ptr(&(pcp2)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) preempt_enable_notrace(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) ret__; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #include <asm-generic/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #endif /* __ARCH_S390_PERCPU__ */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags