Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #ifndef _ASM_X86_ATOMIC_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #define _ASM_X86_ATOMIC_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/cmpxchg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/rmwcc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * Atomic operations that C can't guarantee us.  Useful for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * resource counting etc..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * arch_atomic_read - read atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  * Atomically reads the value of @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) static __always_inline int arch_atomic_read(const atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	 * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	 * it's non-inlined function that increases binary size and stack usage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	return __READ_ONCE((v)->counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  * arch_atomic_set - set atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * @i: required value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * Atomically sets the value of @v to @i.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) static __always_inline void arch_atomic_set(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	__WRITE_ONCE(v->counter, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * arch_atomic_add - add integer to atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * @i: integer value to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  * Atomically adds @i to @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static __always_inline void arch_atomic_add(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	asm volatile(LOCK_PREFIX "addl %1,%0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		     : "+m" (v->counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		     : "ir" (i) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * arch_atomic_sub - subtract integer from atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  * @i: integer value to subtract
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63)  * Atomically subtracts @i from @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) static __always_inline void arch_atomic_sub(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	asm volatile(LOCK_PREFIX "subl %1,%0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		     : "+m" (v->counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		     : "ir" (i) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  * arch_atomic_sub_and_test - subtract value from variable and test result
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74)  * @i: integer value to subtract
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77)  * Atomically subtracts @i from @v and returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78)  * true if the result is zero, or false for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79)  * other cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) #define arch_atomic_sub_and_test arch_atomic_sub_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88)  * arch_atomic_inc - increment atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91)  * Atomically increments @v by 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) static __always_inline void arch_atomic_inc(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	asm volatile(LOCK_PREFIX "incl %0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		     : "+m" (v->counter) :: "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) #define arch_atomic_inc arch_atomic_inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)  * arch_atomic_dec - decrement atomic variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)  * Atomically decrements @v by 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static __always_inline void arch_atomic_dec(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	asm volatile(LOCK_PREFIX "decl %0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		     : "+m" (v->counter) :: "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define arch_atomic_dec arch_atomic_dec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  * arch_atomic_dec_and_test - decrement and test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  * Atomically decrements @v by 1 and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * returns true if the result is 0, or false for all other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define arch_atomic_dec_and_test arch_atomic_dec_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)  * arch_atomic_inc_and_test - increment and test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)  * Atomically increments @v by 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)  * and returns true if the result is zero, or false for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)  * other cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define arch_atomic_inc_and_test arch_atomic_inc_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)  * arch_atomic_add_negative - add and test if negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)  * @i: integer value to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)  * Atomically adds @i to @v and returns true
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)  * if the result is negative, or false when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)  * result is greater than or equal to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define arch_atomic_add_negative arch_atomic_add_negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)  * arch_atomic_add_return - add integer and return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)  * @i: integer value to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)  * Atomically adds @i to @v and returns @i + @v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	return i + xadd(&v->counter, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) #define arch_atomic_add_return arch_atomic_add_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)  * arch_atomic_sub_return - subtract integer and return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)  * @v: pointer of type atomic_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)  * @i: integer value to subtract
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)  * Atomically subtracts @i from @v and returns @v - @i
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	return arch_atomic_add_return(-i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #define arch_atomic_sub_return arch_atomic_sub_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	return xadd(&v->counter, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #define arch_atomic_fetch_add arch_atomic_fetch_add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	return xadd(&v->counter, -i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #define arch_atomic_fetch_sub arch_atomic_fetch_sub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	return arch_cmpxchg(&v->counter, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define arch_atomic_cmpxchg arch_atomic_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	return try_cmpxchg(&v->counter, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static __always_inline int arch_atomic_xchg(atomic_t *v, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	return arch_xchg(&v->counter, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #define arch_atomic_xchg arch_atomic_xchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static __always_inline void arch_atomic_and(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	asm volatile(LOCK_PREFIX "andl %1,%0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 			: "+m" (v->counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 			: "ir" (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 			: "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static __always_inline int arch_atomic_fetch_and(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	int val = arch_atomic_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	do { } while (!arch_atomic_try_cmpxchg(v, &val, val & i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #define arch_atomic_fetch_and arch_atomic_fetch_and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static __always_inline void arch_atomic_or(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	asm volatile(LOCK_PREFIX "orl %1,%0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			: "+m" (v->counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 			: "ir" (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			: "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static __always_inline int arch_atomic_fetch_or(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	int val = arch_atomic_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	do { } while (!arch_atomic_try_cmpxchg(v, &val, val | i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #define arch_atomic_fetch_or arch_atomic_fetch_or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static __always_inline void arch_atomic_xor(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	asm volatile(LOCK_PREFIX "xorl %1,%0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 			: "+m" (v->counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 			: "ir" (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 			: "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	int val = arch_atomic_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	do { } while (!arch_atomic_try_cmpxchg(v, &val, val ^ i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #define arch_atomic_fetch_xor arch_atomic_fetch_xor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) # include <asm/atomic64_32.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) # include <asm/atomic64_64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #define ARCH_ATOMIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #endif /* _ASM_X86_ATOMIC_H */