Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Copyright (2004) Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Copyright (2004, 2005) Ingo Molnar
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * This file contains the spinlock/rwlock implementations for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * Note that some architectures have special knowledge about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * stack frames of these functions in their profile_pc. If you
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * change anything significant here that could change the stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * frame contact the architecture maintainers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <linux/debug_locks.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #ifdef CONFIG_MMIOWB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #ifndef arch_mmiowb_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) DEFINE_PER_CPU(struct mmiowb_state, __mmiowb_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) EXPORT_PER_CPU_SYMBOL(__mmiowb_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  * If lockdep is enabled then we use the non-preemption spin-ops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * The __lock_function inlines are taken from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * spinlock : include/linux/spinlock_api_smp.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  * rwlock   : include/linux/rwlock_api_smp.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * Some architectures can relax in favour of the CPU owning the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #ifndef arch_read_relax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) # define arch_read_relax(l)	cpu_relax()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #ifndef arch_write_relax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) # define arch_write_relax(l)	cpu_relax()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #ifndef arch_spin_relax
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) # define arch_spin_relax(l)	cpu_relax()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * We build the __lock_function inlines here. They are too large for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  * inlining all over the place, but here is only one user per function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  * which embedds them into the calling _lock_function below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63)  * This could be a long-held lock. We both prepare to spin for a long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64)  * time (making _this_ CPU preemptable if possible), and we also signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65)  * towards that other CPU that it should break the lock ASAP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) #define BUILD_LOCK_OPS(op, locktype)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) void __lockfunc __raw_##op##_lock(locktype##_t *lock)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	for (;;) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		preempt_disable();					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		if (likely(do_raw_##op##_trylock(lock)))		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 			break;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		preempt_enable();					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 		arch_##op##_relax(&lock->raw_lock);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) }									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	unsigned long flags;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	for (;;) {							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		preempt_disable();					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		local_irq_save(flags);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 		if (likely(do_raw_##op##_trylock(lock)))		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 			break;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		local_irq_restore(flags);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		preempt_enable();					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		arch_##op##_relax(&lock->raw_lock);			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	}								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	return flags;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) }									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	_raw_##op##_lock_irqsave(lock);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock)		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	unsigned long flags;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	/*							*/	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	/* Careful: we must exclude softirqs too, hence the	*/	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	/* irq-disabling. We use the generic preemption-aware	*/	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	/* function:						*/	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	/**/								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	flags = _raw_##op##_lock_irqsave(lock);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	local_bh_disable();						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	local_irq_restore(flags);					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * Build preemption-friendly versions of the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * lock-spinning functions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  *         __[spin|read|write]_lock()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  *         __[spin|read|write]_lock_irq()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  *         __[spin|read|write]_lock_irqsave()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  *         __[spin|read|write]_lock_bh()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) BUILD_LOCK_OPS(spin, raw_spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) BUILD_LOCK_OPS(read, rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) BUILD_LOCK_OPS(write, rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #ifndef CONFIG_INLINE_SPIN_TRYLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	return __raw_spin_trylock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) EXPORT_SYMBOL(_raw_spin_trylock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	return __raw_spin_trylock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) EXPORT_SYMBOL(_raw_spin_trylock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #ifndef CONFIG_INLINE_SPIN_LOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) void __lockfunc _raw_spin_lock(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	__raw_spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) EXPORT_SYMBOL(_raw_spin_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	return __raw_spin_lock_irqsave(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) EXPORT_SYMBOL(_raw_spin_lock_irqsave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	__raw_spin_lock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) EXPORT_SYMBOL(_raw_spin_lock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #ifndef CONFIG_INLINE_SPIN_LOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	__raw_spin_lock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) EXPORT_SYMBOL(_raw_spin_lock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #ifdef CONFIG_UNINLINE_SPIN_UNLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	__raw_spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) EXPORT_SYMBOL(_raw_spin_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	__raw_spin_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) EXPORT_SYMBOL(_raw_spin_unlock_irqrestore);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	__raw_spin_unlock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) EXPORT_SYMBOL(_raw_spin_unlock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	__raw_spin_unlock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) EXPORT_SYMBOL(_raw_spin_unlock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #ifndef CONFIG_INLINE_READ_TRYLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) int __lockfunc _raw_read_trylock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	return __raw_read_trylock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) EXPORT_SYMBOL(_raw_read_trylock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) #ifndef CONFIG_INLINE_READ_LOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) void __lockfunc _raw_read_lock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	__raw_read_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) EXPORT_SYMBOL(_raw_read_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	return __raw_read_lock_irqsave(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) EXPORT_SYMBOL(_raw_read_lock_irqsave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #ifndef CONFIG_INLINE_READ_LOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) void __lockfunc _raw_read_lock_irq(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	__raw_read_lock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) EXPORT_SYMBOL(_raw_read_lock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #ifndef CONFIG_INLINE_READ_LOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) void __lockfunc _raw_read_lock_bh(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	__raw_read_lock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) EXPORT_SYMBOL(_raw_read_lock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #ifndef CONFIG_INLINE_READ_UNLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) void __lockfunc _raw_read_unlock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	__raw_read_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) EXPORT_SYMBOL(_raw_read_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	__raw_read_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) EXPORT_SYMBOL(_raw_read_unlock_irqrestore);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) void __lockfunc _raw_read_unlock_irq(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	__raw_read_unlock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) EXPORT_SYMBOL(_raw_read_unlock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #ifndef CONFIG_INLINE_READ_UNLOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) void __lockfunc _raw_read_unlock_bh(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	__raw_read_unlock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) EXPORT_SYMBOL(_raw_read_unlock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #ifndef CONFIG_INLINE_WRITE_TRYLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) int __lockfunc _raw_write_trylock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	return __raw_write_trylock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) EXPORT_SYMBOL(_raw_write_trylock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #ifndef CONFIG_INLINE_WRITE_LOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) void __lockfunc _raw_write_lock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	__raw_write_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) EXPORT_SYMBOL(_raw_write_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	return __raw_write_lock_irqsave(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) EXPORT_SYMBOL(_raw_write_lock_irqsave);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) void __lockfunc _raw_write_lock_irq(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	__raw_write_lock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) EXPORT_SYMBOL(_raw_write_lock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #ifndef CONFIG_INLINE_WRITE_LOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) void __lockfunc _raw_write_lock_bh(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	__raw_write_lock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) EXPORT_SYMBOL(_raw_write_lock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #ifndef CONFIG_INLINE_WRITE_UNLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) void __lockfunc _raw_write_unlock(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	__raw_write_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) EXPORT_SYMBOL(_raw_write_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	__raw_write_unlock_irqrestore(lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) EXPORT_SYMBOL(_raw_write_unlock_irqrestore);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) void __lockfunc _raw_write_unlock_irq(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	__raw_write_unlock_irq(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) EXPORT_SYMBOL(_raw_write_unlock_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) void __lockfunc _raw_write_unlock_bh(rwlock_t *lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	__raw_write_unlock_bh(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) EXPORT_SYMBOL(_raw_write_unlock_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) #ifdef CONFIG_DEBUG_LOCK_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) EXPORT_SYMBOL(_raw_spin_lock_nested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 						   int subclass)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 				do_raw_spin_lock_flags, &flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	return flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 				     struct lockdep_map *nest_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) EXPORT_SYMBOL(_raw_spin_lock_nest_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) notrace int in_lock_functions(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	/* Linker adds these: start and end of __lockfunc functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	extern char __lock_text_start[], __lock_text_end[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	return addr >= (unsigned long)__lock_text_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	&& addr < (unsigned long)__lock_text_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) EXPORT_SYMBOL(in_lock_functions);