Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/lockdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/percpu-rwsem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/sched/task.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 			const char *name, struct lock_class_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 	sem->read_count = alloc_percpu(int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 	if (unlikely(!sem->read_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	rcu_sync_init(&sem->rss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	rcuwait_init(&sem->writer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	init_waitqueue_head(&sem->waiters);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	atomic_set(&sem->block, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #ifdef CONFIG_DEBUG_LOCK_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	lockdep_init_map(&sem->dep_map, name, key, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) EXPORT_SYMBOL_GPL(__percpu_init_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) void percpu_free_rwsem(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	 * XXX: temporary kludge. The error path in alloc_super()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	 * assumes that percpu_free_rwsem() is safe after kzalloc().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	if (!sem->read_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	rcu_sync_dtor(&sem->rss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	free_percpu(sem->read_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	sem->read_count = NULL; /* catch use after free bugs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) EXPORT_SYMBOL_GPL(percpu_free_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	this_cpu_inc(*sem->read_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	 * Due to having preemption disabled the decrement happens on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	 * the same CPU as the increment, avoiding the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	 * increment-on-one-CPU-and-decrement-on-another problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	 * If the reader misses the writer's assignment of sem->block, then the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	 * writer is guaranteed to see the reader's increment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	 * Conversely, any readers that increment their sem->read_count after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	 * the writer looks are guaranteed to see the sem->block value, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	 * in turn means that they are guaranteed to immediately decrement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	 * their sem->read_count, so that it doesn't matter that the writer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	 * missed them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	smp_mb(); /* A matches D */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	 * If !sem->block the critical section starts here, matched by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	 * release in percpu_up_write().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	if (likely(!atomic_read_acquire(&sem->block)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 		return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	this_cpu_dec(*sem->read_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	/* Prod writer to re-evaluate readers_active_check() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	rcuwait_wake_up(&sem->writer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	if (atomic_read(&sem->block))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	return atomic_xchg(&sem->block, 1) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	if (reader) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		ret = __percpu_down_read_trylock(sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	return __percpu_down_write_trylock(sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)  * The return value of wait_queue_entry::func means:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  *  <0 - error, wakeup is terminated and the error is returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  *   0 - no wakeup, a next waiter is tried
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  *  >0 - woken, if EXCLUSIVE, counted towards @nr_exclusive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * We use EXCLUSIVE for both readers and writers to preserve FIFO order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  * and play games with the return value to allow waking multiple readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * Specifically, we wake readers until we've woken a single writer, or until a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * trylock fails.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static int percpu_rwsem_wake_function(struct wait_queue_entry *wq_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 				      unsigned int mode, int wake_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 				      void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	bool reader = wq_entry->flags & WQ_FLAG_CUSTOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct percpu_rw_semaphore *sem = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	struct task_struct *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	/* concurrent against percpu_down_write(), can get stolen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	if (!__percpu_rwsem_trylock(sem, reader))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	p = get_task_struct(wq_entry->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	list_del_init(&wq_entry->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	smp_store_release(&wq_entry->private, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	wake_up_process(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	put_task_struct(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	return !reader; /* wake (readers until) 1 writer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	DEFINE_WAIT_FUNC(wq_entry, percpu_rwsem_wake_function);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	bool wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	spin_lock_irq(&sem->waiters.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	 * Serialize against the wakeup in percpu_up_write(), if we fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	 * the trylock, the wakeup must see us on the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	wait = !__percpu_rwsem_trylock(sem, reader);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	if (wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		wq_entry.flags |= WQ_FLAG_EXCLUSIVE | reader * WQ_FLAG_CUSTOM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		__add_wait_queue_entry_tail(&sem->waiters, &wq_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	spin_unlock_irq(&sem->waiters.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	while (wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 		set_current_state(TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		if (!smp_load_acquire(&wq_entry.private))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 		schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	__set_current_state(TASK_RUNNING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) bool __percpu_down_read(struct percpu_rw_semaphore *sem, bool try)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	if (__percpu_down_read_trylock(sem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 		return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	if (try)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	percpu_rwsem_wait(sem, /* .reader = */ true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) EXPORT_SYMBOL_GPL(__percpu_down_read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #define per_cpu_sum(var)						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) ({									\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	typeof(var) __sum = 0;						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	int cpu;							\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	compiletime_assert_atomic_type(__sum);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	for_each_possible_cpu(cpu)					\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		__sum += per_cpu(var, cpu);				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	__sum;								\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)  * Return true if the modular sum of the sem->read_count per-CPU variable is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)  * zero.  If this sum is zero, then it is stable due to the fact that if any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)  * newly arriving readers increment a given counter, they will immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)  * decrement that same counter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  * Assumes sem->block is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static bool readers_active_check(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	if (per_cpu_sum(*sem->read_count) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	 * If we observed the decrement; ensure we see the entire critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	 * section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	smp_mb(); /* C matches B */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) void percpu_down_write(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	/* Notify readers to take the slow path. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	rcu_sync_enter(&sem->rss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	 * Try set sem->block; this provides writer-writer exclusion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	 * Having sem->block set makes new readers block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	if (!__percpu_down_write_trylock(sem))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		percpu_rwsem_wait(sem, /* .reader = */ false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	/* smp_mb() implied by __percpu_down_write_trylock() on success -- D matches A */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	 * If they don't see our store of sem->block, then we are guaranteed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	 * see their sem->read_count increment, and therefore will wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	 * them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	/* Wait for all active readers to complete. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) EXPORT_SYMBOL_GPL(percpu_down_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) void percpu_up_write(struct percpu_rw_semaphore *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	rwsem_release(&sem->dep_map, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	 * Signal the writer is done, no fast path yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	 * One reason that we cannot just immediately flip to readers_fast is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	 * that new readers might fail to see the results of this writer's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	 * critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	 * Therefore we force it through the slow path which guarantees an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	 * acquire and thereby guarantees the critical section's consistency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	atomic_set_release(&sem->block, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	 * Prod any pending reader/writer to make progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	__wake_up(&sem->waiters, TASK_NORMAL, 1, sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	 * Once this completes (at least one RCU-sched grace period hence) the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	 * reader fast path will be available again. Safe to use outside the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	 * exclusive write lock because its counting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	rcu_sync_exit(&sem->rss);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) EXPORT_SYMBOL_GPL(percpu_up_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static LIST_HEAD(destroy_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static DEFINE_SPINLOCK(destroy_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void destroy_list_workfn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	struct percpu_rw_semaphore_atomic *sem, *sem2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	LIST_HEAD(to_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	spin_lock(&destroy_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	list_splice_init(&destroy_list, &to_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	spin_unlock(&destroy_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	if (list_empty(&to_destroy))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	list_for_each_entry_safe(sem, sem2, &to_destroy, destroy_list_entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		percpu_free_rwsem(&sem->rw_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		kfree(sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static DECLARE_WORK(destroy_list_work, destroy_list_workfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) void percpu_rwsem_async_destroy(struct percpu_rw_semaphore_atomic *sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	spin_lock(&destroy_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	list_add_tail(&sem->destroy_list_entry, &destroy_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	spin_unlock(&destroy_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	schedule_work(&destroy_list_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }