Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Based on arch/arm/include/asm/cmpxchg.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2012 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #ifndef __ASM_CMPXCHG_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #define __ASM_CMPXCHG_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/build_bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/lse.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * We need separate acquire parameters for ll/sc and lse, since the full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * barrier case is generated as release+dmb for the former and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * acquire+release for the latter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) {										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	u##sz ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	unsigned long tmp;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	asm volatile(ARM64_LSE_ATOMIC_INSN(					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	/* LL/SC */								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	"	prfm	pstl1strm, %2\n"					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	"1:	ld" #acq "xr" #sfx "\t%" #w "0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	"	st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	"	cbnz	%w1, 1b\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	"	" #mb,								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	/* LSE atomics */							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	"	swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n"		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 		__nops(3)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	"	" #nop_lse)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	: "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	: "r" (x)								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	: cl);									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	return ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) __XCHG_CASE(w, b,     ,  8,        ,    ,  ,  ,  ,         )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) __XCHG_CASE(w, h,     , 16,        ,    ,  ,  ,  ,         )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) __XCHG_CASE(w,  ,     , 32,        ,    ,  ,  ,  ,         )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) __XCHG_CASE( ,  ,     , 64,        ,    ,  ,  ,  ,         )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) __XCHG_CASE(w, b, acq_,  8,        ,    , a, a,  , "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) __XCHG_CASE(w, h, acq_, 16,        ,    , a, a,  , "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) __XCHG_CASE(w,  , acq_, 32,        ,    , a, a,  , "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) __XCHG_CASE( ,  , acq_, 64,        ,    , a, a,  , "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) __XCHG_CASE(w, b, rel_,  8,        ,    ,  ,  , l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) __XCHG_CASE(w, h, rel_, 16,        ,    ,  ,  , l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) __XCHG_CASE(w,  , rel_, 32,        ,    ,  ,  , l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) __XCHG_CASE( ,  , rel_, 64,        ,    ,  ,  , l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) __XCHG_CASE(w, b,  mb_,  8, dmb ish, nop,  , a, l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) __XCHG_CASE(w, h,  mb_, 16, dmb ish, nop,  , a, l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) __XCHG_CASE(w,  ,  mb_, 32, dmb ish, nop,  , a, l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) __XCHG_CASE( ,  ,  mb_, 64, dmb ish, nop,  , a, l, "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) #undef __XCHG_CASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define __XCHG_GEN(sfx)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) static __always_inline  unsigned long __xchg##sfx(unsigned long x,	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 					volatile void *ptr,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 					int size)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	case 1:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		return __xchg_case##sfx##_8(x, ptr);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	case 2:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 		return __xchg_case##sfx##_16(x, ptr);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		return __xchg_case##sfx##_32(x, ptr);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 		return __xchg_case##sfx##_64(x, ptr);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	unreachable();							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) __XCHG_GEN()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) __XCHG_GEN(_acq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) __XCHG_GEN(_rel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) __XCHG_GEN(_mb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) #undef __XCHG_GEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) #define __xchg_wrapper(sfx, ptr, x)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	__ret = (__typeof__(*(ptr)))					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		__xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* xchg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define arch_xchg_relaxed(...)	__xchg_wrapper(    , __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define arch_xchg_acquire(...)	__xchg_wrapper(_acq, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define arch_xchg_release(...)	__xchg_wrapper(_rel, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define arch_xchg(...)		__xchg_wrapper( _mb, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define __CMPXCHG_CASE(name, sz)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr,	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 					      u##sz old,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 					      u##sz new)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	return __lse_ll_sc_body(_cmpxchg_case_##name##sz,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 				ptr, old, new);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) __CMPXCHG_CASE(    ,  8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) __CMPXCHG_CASE(    , 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) __CMPXCHG_CASE(    , 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __CMPXCHG_CASE(    , 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) __CMPXCHG_CASE(acq_,  8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __CMPXCHG_CASE(acq_, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) __CMPXCHG_CASE(acq_, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) __CMPXCHG_CASE(acq_, 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) __CMPXCHG_CASE(rel_,  8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) __CMPXCHG_CASE(rel_, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) __CMPXCHG_CASE(rel_, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) __CMPXCHG_CASE(rel_, 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) __CMPXCHG_CASE(mb_,  8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) __CMPXCHG_CASE(mb_, 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) __CMPXCHG_CASE(mb_, 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) __CMPXCHG_CASE(mb_, 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #undef __CMPXCHG_CASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define __CMPXCHG_DBL(name)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static inline long __cmpxchg_double##name(unsigned long old1,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 					 unsigned long old2,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 					 unsigned long new1,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 					 unsigned long new2,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 					 volatile void *ptr)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	return __lse_ll_sc_body(_cmpxchg_double##name, 			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 				old1, old2, new1, new2, ptr);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) __CMPXCHG_DBL(   )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) __CMPXCHG_DBL(_mb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #undef __CMPXCHG_DBL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define __CMPXCHG_GEN(sfx)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static __always_inline unsigned long __cmpxchg##sfx(volatile void *ptr,	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 					   unsigned long old,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 					   unsigned long new,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 					   int size)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	case 1:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 		return __cmpxchg_case##sfx##_8(ptr, old, new);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	case 2:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		return __cmpxchg_case##sfx##_16(ptr, old, new);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 		return __cmpxchg_case##sfx##_32(ptr, old, new);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		return __cmpxchg_case##sfx##_64(ptr, old, new);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	unreachable();							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) __CMPXCHG_GEN()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) __CMPXCHG_GEN(_acq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) __CMPXCHG_GEN(_rel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) __CMPXCHG_GEN(_mb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #undef __CMPXCHG_GEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #define __cmpxchg_wrapper(sfx, ptr, o, n)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	__ret = (__typeof__(*(ptr)))					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		__cmpxchg##sfx((ptr), (unsigned long)(o),		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 				(unsigned long)(n), sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /* cmpxchg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) #define arch_cmpxchg_relaxed(...)	__cmpxchg_wrapper(    , __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #define arch_cmpxchg_acquire(...)	__cmpxchg_wrapper(_acq, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) #define arch_cmpxchg_release(...)	__cmpxchg_wrapper(_rel, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #define arch_cmpxchg(...)		__cmpxchg_wrapper( _mb, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define arch_cmpxchg_local		arch_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* cmpxchg64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #define arch_cmpxchg64_relaxed		arch_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #define arch_cmpxchg64_acquire		arch_cmpxchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define arch_cmpxchg64_release		arch_cmpxchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #define arch_cmpxchg64			arch_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #define arch_cmpxchg64_local		arch_cmpxchg_local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /* cmpxchg_double */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define system_has_cmpxchg_double()     1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #define __cmpxchg_double_check(ptr1, ptr2)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ({										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	if (sizeof(*(ptr1)) != 8)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		BUILD_BUG();							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1);	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) ({										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	int __ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	__cmpxchg_double_check(ptr1, ptr2);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2),	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 				     (unsigned long)(n1), (unsigned long)(n2),	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 				     ptr1);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	__ret;									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) ({										\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	int __ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	__cmpxchg_double_check(ptr1, ptr2);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2),	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 				  (unsigned long)(n1), (unsigned long)(n2),	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 				  ptr1);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	__ret;									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #define __CMPWAIT_CASE(w, sfx, sz)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static inline void __cmpwait_case_##sz(volatile void *ptr,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 				       unsigned long val)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	unsigned long tmp;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	asm volatile(							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	"	sevl\n"							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	"	wfe\n"							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	"	ldxr" #sfx "\t%" #w "[tmp], %[v]\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	"	eor	%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	"	cbnz	%" #w "[tmp], 1f\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	"	wfe\n"							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	"1:"								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	: [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	: [val] "r" (val));						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) __CMPWAIT_CASE(w, b, 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) __CMPWAIT_CASE(w, h, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) __CMPWAIT_CASE(w,  , 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) __CMPWAIT_CASE( ,  , 64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) #undef __CMPWAIT_CASE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #define __CMPWAIT_GEN(sfx)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) static __always_inline void __cmpwait##sfx(volatile void *ptr,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 				  unsigned long val,			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 				  int size)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	case 1:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 		return __cmpwait_case##sfx##_8(ptr, (u8)val);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	case 2:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		return __cmpwait_case##sfx##_16(ptr, (u16)val);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		return __cmpwait_case##sfx##_32(ptr, val);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		return __cmpwait_case##sfx##_64(ptr, val);		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	unreachable();							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) __CMPWAIT_GEN()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) #undef __CMPWAIT_GEN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #define __cmpwait_relaxed(ptr, val) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	__cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #endif	/* __ASM_CMPXCHG_H */