^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) // Generated by scripts/atomic/gen-atomic-instrumented.sh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) // DO NOT MODIFY THIS FILE DIRECTLY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This file provides wrappers with KASAN instrumentation for atomic operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * To use this functionality an arch's atomic.h file needs to define all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * this file at the end. This file provides atomic_read() that forwards to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * arch_atomic_read() for actual atomic operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Note: if an arch atomic operation is implemented by means of other atomic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * double instrumentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/build_bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/instrumented.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) atomic_read(const atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) instrument_atomic_read(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) return arch_atomic_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define atomic_read atomic_read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #if defined(arch_atomic_read_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) atomic_read_acquire(const atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) instrument_atomic_read(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) return arch_atomic_read_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define atomic_read_acquire atomic_read_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) atomic_set(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) instrument_atomic_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) arch_atomic_set(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define atomic_set atomic_set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #if defined(arch_atomic_set_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) atomic_set_release(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) instrument_atomic_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) arch_atomic_set_release(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define atomic_set_release atomic_set_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) atomic_add(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) arch_atomic_add(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define atomic_add atomic_add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) atomic_add_return(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return arch_atomic_add_return(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define atomic_add_return atomic_add_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #if defined(arch_atomic_add_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) atomic_add_return_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return arch_atomic_add_return_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define atomic_add_return_acquire atomic_add_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #if defined(arch_atomic_add_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) atomic_add_return_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) return arch_atomic_add_return_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define atomic_add_return_release atomic_add_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #if defined(arch_atomic_add_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) atomic_add_return_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return arch_atomic_add_return_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define atomic_add_return_relaxed atomic_add_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) atomic_fetch_add(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return arch_atomic_fetch_add(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define atomic_fetch_add atomic_fetch_add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #if defined(arch_atomic_fetch_add_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) atomic_fetch_add_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return arch_atomic_fetch_add_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define atomic_fetch_add_acquire atomic_fetch_add_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #if defined(arch_atomic_fetch_add_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) atomic_fetch_add_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return arch_atomic_fetch_add_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define atomic_fetch_add_release atomic_fetch_add_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #if defined(arch_atomic_fetch_add_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) atomic_fetch_add_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return arch_atomic_fetch_add_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) atomic_sub(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) arch_atomic_sub(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define atomic_sub atomic_sub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) atomic_sub_return(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return arch_atomic_sub_return(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define atomic_sub_return atomic_sub_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #if defined(arch_atomic_sub_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) atomic_sub_return_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return arch_atomic_sub_return_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #define atomic_sub_return_acquire atomic_sub_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #if defined(arch_atomic_sub_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) atomic_sub_return_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return arch_atomic_sub_return_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #define atomic_sub_return_release atomic_sub_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #if defined(arch_atomic_sub_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) atomic_sub_return_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return arch_atomic_sub_return_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define atomic_sub_return_relaxed atomic_sub_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) atomic_fetch_sub(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return arch_atomic_fetch_sub(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #define atomic_fetch_sub atomic_fetch_sub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #if defined(arch_atomic_fetch_sub_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) atomic_fetch_sub_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return arch_atomic_fetch_sub_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) #if defined(arch_atomic_fetch_sub_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) atomic_fetch_sub_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return arch_atomic_fetch_sub_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #define atomic_fetch_sub_release atomic_fetch_sub_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #if defined(arch_atomic_fetch_sub_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) atomic_fetch_sub_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return arch_atomic_fetch_sub_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #if defined(arch_atomic_inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) atomic_inc(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) arch_atomic_inc(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define atomic_inc atomic_inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #if defined(arch_atomic_inc_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) atomic_inc_return(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return arch_atomic_inc_return(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #define atomic_inc_return atomic_inc_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #if defined(arch_atomic_inc_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) atomic_inc_return_acquire(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return arch_atomic_inc_return_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) #define atomic_inc_return_acquire atomic_inc_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #if defined(arch_atomic_inc_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) atomic_inc_return_release(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return arch_atomic_inc_return_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) #define atomic_inc_return_release atomic_inc_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #if defined(arch_atomic_inc_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) atomic_inc_return_relaxed(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return arch_atomic_inc_return_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #define atomic_inc_return_relaxed atomic_inc_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) #if defined(arch_atomic_fetch_inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) atomic_fetch_inc(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return arch_atomic_fetch_inc(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #define atomic_fetch_inc atomic_fetch_inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #if defined(arch_atomic_fetch_inc_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) atomic_fetch_inc_acquire(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return arch_atomic_fetch_inc_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #if defined(arch_atomic_fetch_inc_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) atomic_fetch_inc_release(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return arch_atomic_fetch_inc_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #define atomic_fetch_inc_release atomic_fetch_inc_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #if defined(arch_atomic_fetch_inc_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) atomic_fetch_inc_relaxed(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return arch_atomic_fetch_inc_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #if defined(arch_atomic_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) atomic_dec(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) arch_atomic_dec(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) #define atomic_dec atomic_dec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #if defined(arch_atomic_dec_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) atomic_dec_return(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) return arch_atomic_dec_return(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #define atomic_dec_return atomic_dec_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #if defined(arch_atomic_dec_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) atomic_dec_return_acquire(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return arch_atomic_dec_return_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #define atomic_dec_return_acquire atomic_dec_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) #if defined(arch_atomic_dec_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) atomic_dec_return_release(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return arch_atomic_dec_return_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #define atomic_dec_return_release atomic_dec_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) #if defined(arch_atomic_dec_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) atomic_dec_return_relaxed(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return arch_atomic_dec_return_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) #define atomic_dec_return_relaxed atomic_dec_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) #if defined(arch_atomic_fetch_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) atomic_fetch_dec(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return arch_atomic_fetch_dec(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #define atomic_fetch_dec atomic_fetch_dec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #if defined(arch_atomic_fetch_dec_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) atomic_fetch_dec_acquire(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) return arch_atomic_fetch_dec_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #if defined(arch_atomic_fetch_dec_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) atomic_fetch_dec_release(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return arch_atomic_fetch_dec_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #define atomic_fetch_dec_release atomic_fetch_dec_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) #if defined(arch_atomic_fetch_dec_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) atomic_fetch_dec_relaxed(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return arch_atomic_fetch_dec_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) atomic_and(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) arch_atomic_and(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) #define atomic_and atomic_and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) atomic_fetch_and(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return arch_atomic_fetch_and(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) #define atomic_fetch_and atomic_fetch_and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) #if defined(arch_atomic_fetch_and_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) atomic_fetch_and_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return arch_atomic_fetch_and_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) #define atomic_fetch_and_acquire atomic_fetch_and_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) #if defined(arch_atomic_fetch_and_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) atomic_fetch_and_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return arch_atomic_fetch_and_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) #define atomic_fetch_and_release atomic_fetch_and_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) #if defined(arch_atomic_fetch_and_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) atomic_fetch_and_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) return arch_atomic_fetch_and_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) #if defined(arch_atomic_andnot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) atomic_andnot(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) arch_atomic_andnot(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) #define atomic_andnot atomic_andnot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) #if defined(arch_atomic_fetch_andnot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) atomic_fetch_andnot(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return arch_atomic_fetch_andnot(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) #define atomic_fetch_andnot atomic_fetch_andnot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) #if defined(arch_atomic_fetch_andnot_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) atomic_fetch_andnot_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return arch_atomic_fetch_andnot_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) #if defined(arch_atomic_fetch_andnot_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) atomic_fetch_andnot_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) return arch_atomic_fetch_andnot_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) #define atomic_fetch_andnot_release atomic_fetch_andnot_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) #if defined(arch_atomic_fetch_andnot_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) atomic_fetch_andnot_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return arch_atomic_fetch_andnot_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) atomic_or(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) arch_atomic_or(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) #define atomic_or atomic_or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) #if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) atomic_fetch_or(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return arch_atomic_fetch_or(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) #define atomic_fetch_or atomic_fetch_or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) #if defined(arch_atomic_fetch_or_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) atomic_fetch_or_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return arch_atomic_fetch_or_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) #define atomic_fetch_or_acquire atomic_fetch_or_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) #if defined(arch_atomic_fetch_or_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) atomic_fetch_or_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) return arch_atomic_fetch_or_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) #define atomic_fetch_or_release atomic_fetch_or_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) #if defined(arch_atomic_fetch_or_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) atomic_fetch_or_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return arch_atomic_fetch_or_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) atomic_xor(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) arch_atomic_xor(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) #define atomic_xor atomic_xor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) #if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) atomic_fetch_xor(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return arch_atomic_fetch_xor(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) #define atomic_fetch_xor atomic_fetch_xor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) #if defined(arch_atomic_fetch_xor_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) atomic_fetch_xor_acquire(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) return arch_atomic_fetch_xor_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) #if defined(arch_atomic_fetch_xor_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) atomic_fetch_xor_release(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return arch_atomic_fetch_xor_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) #define atomic_fetch_xor_release atomic_fetch_xor_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) #if defined(arch_atomic_fetch_xor_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) atomic_fetch_xor_relaxed(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return arch_atomic_fetch_xor_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) #if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) atomic_xchg(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return arch_atomic_xchg(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) #define atomic_xchg atomic_xchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) #if defined(arch_atomic_xchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) atomic_xchg_acquire(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return arch_atomic_xchg_acquire(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) #define atomic_xchg_acquire atomic_xchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) #if defined(arch_atomic_xchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) atomic_xchg_release(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) return arch_atomic_xchg_release(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) #define atomic_xchg_release atomic_xchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) #if defined(arch_atomic_xchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) atomic_xchg_relaxed(atomic_t *v, int i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return arch_atomic_xchg_relaxed(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) #define atomic_xchg_relaxed atomic_xchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) atomic_cmpxchg(atomic_t *v, int old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return arch_atomic_cmpxchg(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) #define atomic_cmpxchg atomic_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) #if defined(arch_atomic_cmpxchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return arch_atomic_cmpxchg_acquire(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) #if defined(arch_atomic_cmpxchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) atomic_cmpxchg_release(atomic_t *v, int old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return arch_atomic_cmpxchg_release(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) #define atomic_cmpxchg_release atomic_cmpxchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) #if defined(arch_atomic_cmpxchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return arch_atomic_cmpxchg_relaxed(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) #if defined(arch_atomic_try_cmpxchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) atomic_try_cmpxchg(atomic_t *v, int *old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) return arch_atomic_try_cmpxchg(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) #define atomic_try_cmpxchg atomic_try_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) #if defined(arch_atomic_try_cmpxchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) return arch_atomic_try_cmpxchg_acquire(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) #if defined(arch_atomic_try_cmpxchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return arch_atomic_try_cmpxchg_release(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #if defined(arch_atomic_try_cmpxchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) return arch_atomic_try_cmpxchg_relaxed(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) #if defined(arch_atomic_sub_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) atomic_sub_and_test(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return arch_atomic_sub_and_test(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) #define atomic_sub_and_test atomic_sub_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) #if defined(arch_atomic_dec_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) atomic_dec_and_test(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return arch_atomic_dec_and_test(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) #define atomic_dec_and_test atomic_dec_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) #if defined(arch_atomic_inc_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) atomic_inc_and_test(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return arch_atomic_inc_and_test(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) #define atomic_inc_and_test atomic_inc_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) #if defined(arch_atomic_add_negative)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) atomic_add_negative(int i, atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) return arch_atomic_add_negative(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) #define atomic_add_negative atomic_add_negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) #if defined(arch_atomic_fetch_add_unless)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) atomic_fetch_add_unless(atomic_t *v, int a, int u)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return arch_atomic_fetch_add_unless(v, a, u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) #define atomic_fetch_add_unless atomic_fetch_add_unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) #if defined(arch_atomic_add_unless)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) atomic_add_unless(atomic_t *v, int a, int u)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) return arch_atomic_add_unless(v, a, u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) #define atomic_add_unless atomic_add_unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) #if defined(arch_atomic_inc_not_zero)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) atomic_inc_not_zero(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return arch_atomic_inc_not_zero(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) #define atomic_inc_not_zero atomic_inc_not_zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) #if defined(arch_atomic_inc_unless_negative)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) atomic_inc_unless_negative(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return arch_atomic_inc_unless_negative(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) #define atomic_inc_unless_negative atomic_inc_unless_negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) #if defined(arch_atomic_dec_unless_positive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) atomic_dec_unless_positive(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return arch_atomic_dec_unless_positive(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) #define atomic_dec_unless_positive atomic_dec_unless_positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) #if defined(arch_atomic_dec_if_positive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) static __always_inline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) atomic_dec_if_positive(atomic_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return arch_atomic_dec_if_positive(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) #define atomic_dec_if_positive atomic_dec_if_positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) atomic64_read(const atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) instrument_atomic_read(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return arch_atomic64_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) #define atomic64_read atomic64_read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) #if defined(arch_atomic64_read_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) atomic64_read_acquire(const atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) instrument_atomic_read(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) return arch_atomic64_read_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) #define atomic64_read_acquire atomic64_read_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) atomic64_set(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) instrument_atomic_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) arch_atomic64_set(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) #define atomic64_set atomic64_set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) #if defined(arch_atomic64_set_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) atomic64_set_release(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) instrument_atomic_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) arch_atomic64_set_release(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) #define atomic64_set_release atomic64_set_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) atomic64_add(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) arch_atomic64_add(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) #define atomic64_add atomic64_add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) #if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) atomic64_add_return(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) return arch_atomic64_add_return(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) #define atomic64_add_return atomic64_add_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) #if defined(arch_atomic64_add_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) atomic64_add_return_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) return arch_atomic64_add_return_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) #define atomic64_add_return_acquire atomic64_add_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) #if defined(arch_atomic64_add_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) atomic64_add_return_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) return arch_atomic64_add_return_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) #define atomic64_add_return_release atomic64_add_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) #if defined(arch_atomic64_add_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) atomic64_add_return_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) return arch_atomic64_add_return_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) #define atomic64_add_return_relaxed atomic64_add_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) #if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) atomic64_fetch_add(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return arch_atomic64_fetch_add(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) #define atomic64_fetch_add atomic64_fetch_add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) #if defined(arch_atomic64_fetch_add_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return arch_atomic64_fetch_add_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #if defined(arch_atomic64_fetch_add_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) atomic64_fetch_add_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) return arch_atomic64_fetch_add_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) #define atomic64_fetch_add_release atomic64_fetch_add_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) #if defined(arch_atomic64_fetch_add_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) return arch_atomic64_fetch_add_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) atomic64_sub(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) arch_atomic64_sub(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) #define atomic64_sub atomic64_sub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) #if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) atomic64_sub_return(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) return arch_atomic64_sub_return(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) #define atomic64_sub_return atomic64_sub_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) #if defined(arch_atomic64_sub_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) atomic64_sub_return_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) return arch_atomic64_sub_return_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) #define atomic64_sub_return_acquire atomic64_sub_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) #if defined(arch_atomic64_sub_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) atomic64_sub_return_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) return arch_atomic64_sub_return_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) #define atomic64_sub_return_release atomic64_sub_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) #if defined(arch_atomic64_sub_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return arch_atomic64_sub_return_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) #if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) atomic64_fetch_sub(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) return arch_atomic64_fetch_sub(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) #define atomic64_fetch_sub atomic64_fetch_sub
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) #if defined(arch_atomic64_fetch_sub_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) return arch_atomic64_fetch_sub_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) #if defined(arch_atomic64_fetch_sub_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) atomic64_fetch_sub_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) return arch_atomic64_fetch_sub_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) #define atomic64_fetch_sub_release atomic64_fetch_sub_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) #if defined(arch_atomic64_fetch_sub_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return arch_atomic64_fetch_sub_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) #if defined(arch_atomic64_inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) atomic64_inc(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) arch_atomic64_inc(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) #define atomic64_inc atomic64_inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) #if defined(arch_atomic64_inc_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) atomic64_inc_return(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) return arch_atomic64_inc_return(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) #define atomic64_inc_return atomic64_inc_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) #if defined(arch_atomic64_inc_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) atomic64_inc_return_acquire(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) return arch_atomic64_inc_return_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) #define atomic64_inc_return_acquire atomic64_inc_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) #if defined(arch_atomic64_inc_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) atomic64_inc_return_release(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return arch_atomic64_inc_return_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) #define atomic64_inc_return_release atomic64_inc_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) #if defined(arch_atomic64_inc_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) atomic64_inc_return_relaxed(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) return arch_atomic64_inc_return_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) #if defined(arch_atomic64_fetch_inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) atomic64_fetch_inc(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) return arch_atomic64_fetch_inc(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) #define atomic64_fetch_inc atomic64_fetch_inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) #if defined(arch_atomic64_fetch_inc_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) atomic64_fetch_inc_acquire(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) return arch_atomic64_fetch_inc_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) #if defined(arch_atomic64_fetch_inc_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) atomic64_fetch_inc_release(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) return arch_atomic64_fetch_inc_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) #define atomic64_fetch_inc_release atomic64_fetch_inc_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) #if defined(arch_atomic64_fetch_inc_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) atomic64_fetch_inc_relaxed(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) return arch_atomic64_fetch_inc_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) #if defined(arch_atomic64_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) atomic64_dec(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) arch_atomic64_dec(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) #define atomic64_dec atomic64_dec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) #if defined(arch_atomic64_dec_return)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) atomic64_dec_return(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) return arch_atomic64_dec_return(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) #define atomic64_dec_return atomic64_dec_return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) #if defined(arch_atomic64_dec_return_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) atomic64_dec_return_acquire(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) return arch_atomic64_dec_return_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) #define atomic64_dec_return_acquire atomic64_dec_return_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) #if defined(arch_atomic64_dec_return_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) atomic64_dec_return_release(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return arch_atomic64_dec_return_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) #define atomic64_dec_return_release atomic64_dec_return_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) #if defined(arch_atomic64_dec_return_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) atomic64_dec_return_relaxed(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) return arch_atomic64_dec_return_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) #if defined(arch_atomic64_fetch_dec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) atomic64_fetch_dec(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) return arch_atomic64_fetch_dec(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) #define atomic64_fetch_dec atomic64_fetch_dec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) #if defined(arch_atomic64_fetch_dec_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) atomic64_fetch_dec_acquire(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) return arch_atomic64_fetch_dec_acquire(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) #if defined(arch_atomic64_fetch_dec_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) atomic64_fetch_dec_release(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return arch_atomic64_fetch_dec_release(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) #define atomic64_fetch_dec_release atomic64_fetch_dec_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) #if defined(arch_atomic64_fetch_dec_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) atomic64_fetch_dec_relaxed(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) return arch_atomic64_fetch_dec_relaxed(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) atomic64_and(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) arch_atomic64_and(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) #define atomic64_and atomic64_and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) #if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) atomic64_fetch_and(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) return arch_atomic64_fetch_and(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) #define atomic64_fetch_and atomic64_fetch_and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) #if defined(arch_atomic64_fetch_and_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) return arch_atomic64_fetch_and_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) #if defined(arch_atomic64_fetch_and_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) atomic64_fetch_and_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) return arch_atomic64_fetch_and_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) #define atomic64_fetch_and_release atomic64_fetch_and_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) #if defined(arch_atomic64_fetch_and_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return arch_atomic64_fetch_and_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) #if defined(arch_atomic64_andnot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) atomic64_andnot(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) arch_atomic64_andnot(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) #define atomic64_andnot atomic64_andnot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) #if defined(arch_atomic64_fetch_andnot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) atomic64_fetch_andnot(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) return arch_atomic64_fetch_andnot(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) #define atomic64_fetch_andnot atomic64_fetch_andnot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) #if defined(arch_atomic64_fetch_andnot_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) return arch_atomic64_fetch_andnot_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) #if defined(arch_atomic64_fetch_andnot_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) return arch_atomic64_fetch_andnot_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) #if defined(arch_atomic64_fetch_andnot_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) return arch_atomic64_fetch_andnot_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) atomic64_or(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) arch_atomic64_or(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) #define atomic64_or atomic64_or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) #if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) atomic64_fetch_or(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) return arch_atomic64_fetch_or(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) #define atomic64_fetch_or atomic64_fetch_or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) #if defined(arch_atomic64_fetch_or_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) return arch_atomic64_fetch_or_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) #if defined(arch_atomic64_fetch_or_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) atomic64_fetch_or_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) return arch_atomic64_fetch_or_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) #define atomic64_fetch_or_release atomic64_fetch_or_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) #if defined(arch_atomic64_fetch_or_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) return arch_atomic64_fetch_or_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) static __always_inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) atomic64_xor(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) arch_atomic64_xor(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) #define atomic64_xor atomic64_xor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) #if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) atomic64_fetch_xor(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) return arch_atomic64_fetch_xor(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) #define atomic64_fetch_xor atomic64_fetch_xor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) #if defined(arch_atomic64_fetch_xor_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) return arch_atomic64_fetch_xor_acquire(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) #if defined(arch_atomic64_fetch_xor_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) atomic64_fetch_xor_release(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) return arch_atomic64_fetch_xor_release(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) #define atomic64_fetch_xor_release atomic64_fetch_xor_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) #if defined(arch_atomic64_fetch_xor_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) return arch_atomic64_fetch_xor_relaxed(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) #if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) atomic64_xchg(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) return arch_atomic64_xchg(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) #define atomic64_xchg atomic64_xchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) #if defined(arch_atomic64_xchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) atomic64_xchg_acquire(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return arch_atomic64_xchg_acquire(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) #define atomic64_xchg_acquire atomic64_xchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) #if defined(arch_atomic64_xchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) atomic64_xchg_release(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return arch_atomic64_xchg_release(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) #define atomic64_xchg_release atomic64_xchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) #if defined(arch_atomic64_xchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) atomic64_xchg_relaxed(atomic64_t *v, s64 i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) return arch_atomic64_xchg_relaxed(v, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) #define atomic64_xchg_relaxed atomic64_xchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) #if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) return arch_atomic64_cmpxchg(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) #define atomic64_cmpxchg atomic64_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) #if defined(arch_atomic64_cmpxchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) return arch_atomic64_cmpxchg_acquire(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) #if defined(arch_atomic64_cmpxchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) return arch_atomic64_cmpxchg_release(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) #define atomic64_cmpxchg_release atomic64_cmpxchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) #if defined(arch_atomic64_cmpxchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) return arch_atomic64_cmpxchg_relaxed(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) #define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) #if defined(arch_atomic64_try_cmpxchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) return arch_atomic64_try_cmpxchg(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) #define atomic64_try_cmpxchg atomic64_try_cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) #if defined(arch_atomic64_try_cmpxchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) return arch_atomic64_try_cmpxchg_acquire(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) #if defined(arch_atomic64_try_cmpxchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) return arch_atomic64_try_cmpxchg_release(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) #if defined(arch_atomic64_try_cmpxchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) instrument_atomic_read_write(old, sizeof(*old));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) #if defined(arch_atomic64_sub_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) atomic64_sub_and_test(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) return arch_atomic64_sub_and_test(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) #define atomic64_sub_and_test atomic64_sub_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) #if defined(arch_atomic64_dec_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) atomic64_dec_and_test(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return arch_atomic64_dec_and_test(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) #define atomic64_dec_and_test atomic64_dec_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) #if defined(arch_atomic64_inc_and_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) atomic64_inc_and_test(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) return arch_atomic64_inc_and_test(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) #define atomic64_inc_and_test atomic64_inc_and_test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) #if defined(arch_atomic64_add_negative)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) atomic64_add_negative(s64 i, atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) return arch_atomic64_add_negative(i, v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) #define atomic64_add_negative atomic64_add_negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) #if defined(arch_atomic64_fetch_add_unless)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) return arch_atomic64_fetch_add_unless(v, a, u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) #define atomic64_fetch_add_unless atomic64_fetch_add_unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) #if defined(arch_atomic64_add_unless)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) return arch_atomic64_add_unless(v, a, u);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) #define atomic64_add_unless atomic64_add_unless
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) #if defined(arch_atomic64_inc_not_zero)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) atomic64_inc_not_zero(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) return arch_atomic64_inc_not_zero(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) #define atomic64_inc_not_zero atomic64_inc_not_zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) #if defined(arch_atomic64_inc_unless_negative)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) atomic64_inc_unless_negative(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) return arch_atomic64_inc_unless_negative(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) #define atomic64_inc_unless_negative atomic64_inc_unless_negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) #if defined(arch_atomic64_dec_unless_positive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) static __always_inline bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) atomic64_dec_unless_positive(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) return arch_atomic64_dec_unless_positive(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) #define atomic64_dec_unless_positive atomic64_dec_unless_positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) #if defined(arch_atomic64_dec_if_positive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) static __always_inline s64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) atomic64_dec_if_positive(atomic64_t *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) instrument_atomic_read_write(v, sizeof(*v));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) return arch_atomic64_dec_if_positive(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) #define atomic64_dec_if_positive atomic64_dec_if_positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) #if !defined(arch_xchg_relaxed) || defined(arch_xchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) #define xchg(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) arch_xchg(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) #if defined(arch_xchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) #define xchg_acquire(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) #if defined(arch_xchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) #define xchg_release(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) arch_xchg_release(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) #if defined(arch_xchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) #define xchg_relaxed(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) #if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) #define cmpxchg(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) #if defined(arch_cmpxchg_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) #define cmpxchg_acquire(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) #if defined(arch_cmpxchg_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) #define cmpxchg_release(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) #if defined(arch_cmpxchg_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) #define cmpxchg_relaxed(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) #if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) #define cmpxchg64(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) #if defined(arch_cmpxchg64_acquire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) #define cmpxchg64_acquire(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) #if defined(arch_cmpxchg64_release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) #define cmpxchg64_release(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) #if defined(arch_cmpxchg64_relaxed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) #define cmpxchg64_relaxed(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) #define cmpxchg_local(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) #define cmpxchg64_local(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) #define sync_cmpxchg(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) #define cmpxchg_double(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) #define cmpxchg_double_local(ptr, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) typeof(ptr) __ai_ptr = (ptr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) // 9d5e6a315fb1335d02f0ccd3655a91c3dafcc63e
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags