Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Copyright (C) 2014 Regents of the University of California
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #ifndef _ASM_RISCV_CMPXCHG_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #define _ASM_RISCV_CMPXCHG_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/fence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #define __xchg_relaxed(ptr, new, size)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	__typeof__(new) __new = (new);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 			"	amoswap.w %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 			"	amoswap.d %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #define xchg_relaxed(ptr, x)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	__typeof__(*(ptr)) _x_ = (x);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	(__typeof__(*(ptr))) __xchg_relaxed((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 					    _x_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #define __xchg_acquire(ptr, new, size)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	__typeof__(new) __new = (new);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 			"	amoswap.w %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 			RISCV_ACQUIRE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 			"	amoswap.d %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 			RISCV_ACQUIRE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #define xchg_acquire(ptr, x)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	__typeof__(*(ptr)) _x_ = (x);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	(__typeof__(*(ptr))) __xchg_acquire((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 					    _x_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) #define __xchg_release(ptr, new, size)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	__typeof__(new) __new = (new);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 			RISCV_RELEASE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 			"	amoswap.w %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 			RISCV_RELEASE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 			"	amoswap.d %0, %2, %1\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define xchg_release(ptr, x)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	__typeof__(*(ptr)) _x_ = (x);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	(__typeof__(*(ptr))) __xchg_release((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 					    _x_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define __xchg(ptr, new, size)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	__typeof__(new) __new = (new);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 			"	amoswap.w.aqrl %0, %2, %1\n"		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 			"	amoswap.d.aqrl %0, %2, %1\n"		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 			: "=r" (__ret), "+A" (*__ptr)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 			: "r" (__new)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define xchg(ptr, x)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	__typeof__(*(ptr)) _x_ = (x);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define xchg32(ptr, x)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	BUILD_BUG_ON(sizeof(*(ptr)) != 4);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	xchg((ptr), (x));						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define xchg64(ptr, x)							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	xchg((ptr), (x));						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)  * store NEW in MEM.  Return the initial value in MEM.  Success is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)  * indicated by comparing RETURN with OLD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #define __cmpxchg_relaxed(ptr, old, new, size)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	__typeof__(*(ptr)) __old = (old);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	__typeof__(*(ptr)) __new = (new);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	register unsigned int __rc;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 			"0:	lr.w %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 			"	bne  %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 			"	sc.w %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 			: "rJ" ((long)__old), "rJ" (__new)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 			"0:	lr.d %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 			"	bne %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 			"	sc.d %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 			: "rJ" (__old), "rJ" (__new)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define cmpxchg_relaxed(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	__typeof__(*(ptr)) _o_ = (o);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	__typeof__(*(ptr)) _n_ = (n);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 					_o_, _n_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define __cmpxchg_acquire(ptr, old, new, size)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	__typeof__(*(ptr)) __old = (old);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	__typeof__(*(ptr)) __new = (new);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	register unsigned int __rc;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 			"0:	lr.w %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 			"	bne  %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 			"	sc.w %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 			RISCV_ACQUIRE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 			: "rJ" ((long)__old), "rJ" (__new)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			"0:	lr.d %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			"	bne %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 			"	sc.d %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 			RISCV_ACQUIRE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 			: "rJ" (__old), "rJ" (__new)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #define cmpxchg_acquire(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	__typeof__(*(ptr)) _o_ = (o);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	__typeof__(*(ptr)) _n_ = (n);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	(__typeof__(*(ptr))) __cmpxchg_acquire((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 					_o_, _n_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #define __cmpxchg_release(ptr, old, new, size)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	__typeof__(*(ptr)) __old = (old);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	__typeof__(*(ptr)) __new = (new);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	register unsigned int __rc;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 			RISCV_RELEASE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 			"0:	lr.w %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 			"	bne  %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 			"	sc.w %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 			: "rJ" ((long)__old), "rJ" (__new)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 			RISCV_RELEASE_BARRIER				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 			"0:	lr.d %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 			"	bne %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 			"	sc.d %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 			: "rJ" (__old), "rJ" (__new)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #define cmpxchg_release(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	__typeof__(*(ptr)) _o_ = (o);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	__typeof__(*(ptr)) _n_ = (n);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	(__typeof__(*(ptr))) __cmpxchg_release((ptr),			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 					_o_, _n_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #define __cmpxchg(ptr, old, new, size)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	__typeof__(ptr) __ptr = (ptr);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	__typeof__(*(ptr)) __old = (old);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	__typeof__(*(ptr)) __new = (new);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	__typeof__(*(ptr)) __ret;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	register unsigned int __rc;					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	switch (size) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	case 4:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 			"0:	lr.w %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 			"	bne  %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 			"	sc.w.rl %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 			"	fence rw, rw\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 			: "rJ" ((long)__old), "rJ" (__new)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	case 8:								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 		__asm__ __volatile__ (					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 			"0:	lr.d %0, %2\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 			"	bne %0, %z3, 1f\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 			"	sc.d.rl %1, %z4, %2\n"			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 			"	bnez %1, 0b\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 			"	fence rw, rw\n"				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 			"1:\n"						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 			: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 			: "rJ" (__old), "rJ" (__new)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 			: "memory");					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		break;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	default:							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		BUILD_BUG();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	__ret;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #define cmpxchg(ptr, o, n)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	__typeof__(*(ptr)) _o_ = (o);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	__typeof__(*(ptr)) _n_ = (n);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 	(__typeof__(*(ptr))) __cmpxchg((ptr),				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 				       _o_, _n_, sizeof(*(ptr)));	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define cmpxchg_local(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #define cmpxchg32(ptr, o, n)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	BUILD_BUG_ON(sizeof(*(ptr)) != 4);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 	cmpxchg((ptr), (o), (n));					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) #define cmpxchg32_local(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	BUILD_BUG_ON(sizeof(*(ptr)) != 4);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	cmpxchg_relaxed((ptr), (o), (n))				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #define cmpxchg64(ptr, o, n)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	cmpxchg((ptr), (o), (n));					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #define cmpxchg64_local(ptr, o, n)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	BUILD_BUG_ON(sizeof(*(ptr)) != 8);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	cmpxchg_relaxed((ptr), (o), (n));				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) #endif /* _ASM_RISCV_CMPXCHG_H */