^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #ifndef __ASM_SPINLOCK_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #define __ASM_SPINLOCK_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/spinlock_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define arch_spin_is_locked(x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #ifdef CONFIG_ARC_HAS_LLSC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) static inline void arch_spin_lock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) unsigned int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) "1: llock %[val], [%[slock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) " breq %[val], %[LOCKED], 1b \n" /* spin while LOCKED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) " scond %[LOCKED], [%[slock]] \n" /* acquire */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) : [val] "=&r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) : [slock] "r" (&(lock->slock)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * ACQUIRE barrier to ensure load/store after taking the lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * don't "bleed-up" out of the critical section (leak-in is allowed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * http://www.spinics.net/lists/kernel/msg2010409.html
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * ARCv2 only has load-load, store-store and all-all barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * thus need the full all-all barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static inline int arch_spin_trylock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) unsigned int val, got_it = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) "1: llock %[val], [%[slock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) " breq %[val], %[LOCKED], 4f \n" /* already LOCKED, just bail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) " scond %[LOCKED], [%[slock]] \n" /* acquire */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) " mov %[got_it], 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) "4: \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) : [val] "=&r" (val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) [got_it] "+&r" (got_it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) : [slock] "r" (&(lock->slock)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) [LOCKED] "r" (__ARCH_SPIN_LOCK_LOCKED__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) return got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static inline void arch_spin_unlock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * Read-write spinlocks, allowing multiple readers but only one writer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Unfair locking as Writers could be starved indefinitely by Reader(s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static inline void arch_read_lock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) unsigned int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * zero means writer holds the lock exclusively, deny Reader.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * Otherwise grant lock to first/subseq reader
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * if (rw->counter > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * rw->counter--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) "1: llock %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) " brls %[val], %[WR_LOCKED], 1b\n" /* <= 0: spin while write locked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) " sub %[val], %[val], 1 \n" /* reader lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) " scond %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) : [val] "=&r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) : [rwlock] "r" (&(rw->counter)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) [WR_LOCKED] "ir" (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static inline int arch_read_trylock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) unsigned int val, got_it = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) "1: llock %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) " brls %[val], %[WR_LOCKED], 4f\n" /* <= 0: already write locked, bail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) " sub %[val], %[val], 1 \n" /* counter-- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) " scond %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) " bnz 1b \n" /* retry if collided with someone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) " mov %[got_it], 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) "4: ; --- done --- \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) : [val] "=&r" (val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) [got_it] "+&r" (got_it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) : [rwlock] "r" (&(rw->counter)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) [WR_LOCKED] "ir" (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static inline void arch_write_lock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) unsigned int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * deny writer. Otherwise if unlocked grant to writer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * Hence the claim that Linux rwlocks are unfair to writers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * (can be starved for an indefinite time by readers).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * rw->counter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) "1: llock %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) " brne %[val], %[UNLOCKED], 1b \n" /* while !UNLOCKED spin */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) " mov %[val], %[WR_LOCKED] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) " scond %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) : [val] "=&r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) : [rwlock] "r" (&(rw->counter)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) [WR_LOCKED] "ir" (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline int arch_write_trylock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned int val, got_it = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) "1: llock %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) " brne %[val], %[UNLOCKED], 4f \n" /* !UNLOCKED, bail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) " mov %[val], %[WR_LOCKED] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) " scond %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) " bnz 1b \n" /* retry if collided with someone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) " mov %[got_it], 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) "4: ; --- done --- \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) : [val] "=&r" (val),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) [got_it] "+&r" (got_it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) : [rwlock] "r" (&(rw->counter)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) [UNLOCKED] "ir" (__ARCH_RW_LOCK_UNLOCKED__),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) [WR_LOCKED] "ir" (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static inline void arch_read_unlock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) unsigned int val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * rw->counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) "1: llock %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) " add %[val], %[val], 1 \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) " scond %[val], [%[rwlock]] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) " bnz 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) " \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) : [val] "=&r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) : [rwlock] "r" (&(rw->counter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) : "memory", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static inline void arch_write_unlock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) #else /* !CONFIG_ARC_HAS_LLSC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static inline void arch_spin_lock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * Per lkmm, smp_mb() is only required after _lock (and before_unlock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * for ACQ and REL semantics respectively. However EX based spinlocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * need the extra smp_mb to workaround a hardware quirk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) "1: ex %0, [%1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) " breq %0, %2, 1b \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) : "+&r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static inline int arch_spin_trylock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) "1: ex %0, [%1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) : "+r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) : "r"(&(lock->slock))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static inline void arch_spin_unlock(arch_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * RELEASE barrier: given the instructions avail on ARCv2, full barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * is the only option
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * EX is not really required here, a simple STore of 0 suffices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * However this causes tasklist livelocks in SystemC based SMP virtual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * platforms where the systemc core scheduler uses EX as a cue for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * moving to next core. Do a git log of this file for details
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) " ex %0, [%1] \n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) : "+r" (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) : "r"(&(lock->slock))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * see pairing version/comment in arch_spin_lock above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * Read-write spinlocks, allowing multiple readers but only one writer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * Unfair locking as Writers could be starved indefinitely by Reader(s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * The spinlock itself is contained in @counter and access to it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * serialized with @lock_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static inline int arch_read_trylock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) arch_spin_lock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * zero means writer holds the lock exclusively, deny Reader.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * Otherwise grant lock to first/subseq reader
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (rw->counter > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) rw->counter--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) arch_spin_unlock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /* 1 - lock taken successfully */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static inline int arch_write_trylock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) arch_spin_lock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * deny writer. Otherwise if unlocked grant to writer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * Hence the claim that Linux rwlocks are unfair to writers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * (can be starved for an indefinite time by readers).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) rw->counter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) arch_spin_unlock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static inline void arch_read_lock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) while (!arch_read_trylock(rw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static inline void arch_write_lock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) while (!arch_write_trylock(rw))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static inline void arch_read_unlock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) arch_spin_lock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) rw->counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) arch_spin_unlock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) static inline void arch_write_unlock(arch_rwlock_t *rw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) arch_spin_lock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) arch_spin_unlock(&(rw->lock_mutex));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) #endif /* __ASM_SPINLOCK_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags