^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef __ASM_PREEMPT_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define __ASM_PREEMPT_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <asm/rmwcc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <asm/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) DECLARE_PER_CPU(int, __preempt_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) /* We use the MSB mostly because its available */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define PREEMPT_NEED_RESCHED 0x80000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * that a decrement hitting 0 means we can and should reschedule.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * that think a non-zero value indicates we cannot preempt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static __always_inline int preempt_count(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static __always_inline void preempt_count_set(int pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int old, new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) old = raw_cpu_read_4(__preempt_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) new = (old & PREEMPT_NEED_RESCHED) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) (pc & ~PREEMPT_NEED_RESCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) } while (raw_cpu_cmpxchg_4(__preempt_count, old, new) != old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * must be macros to avoid header recursion hell
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define init_task_preempt_count(p) do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define init_idle_preempt_count(p, cpu) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * We fold the NEED_RESCHED bit into the preempt count such that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * preempt_enable() can decrement and test for needing to reschedule with a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * single instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * We invert the actual bit, so that when the decrement hits 0 we know we both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * need to resched (the bit is cleared) and can resched (no preempt count).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static __always_inline void set_preempt_need_resched(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static __always_inline void clear_preempt_need_resched(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static __always_inline bool test_preempt_need_resched(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * The various preempt_count add/sub methods
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static __always_inline void __preempt_count_add(int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) raw_cpu_add_4(__preempt_count, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static __always_inline void __preempt_count_sub(int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) raw_cpu_add_4(__preempt_count, -val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * a decrement which hits zero means we have no preempt_count and should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * reschedule.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static __always_inline bool __preempt_count_dec_and_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * Returns true when we need to resched and can (barring IRQ state).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static __always_inline bool should_resched(int preempt_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #ifdef CONFIG_PREEMPTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) extern asmlinkage void preempt_schedule_thunk(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) # define __preempt_schedule() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) extern asmlinkage void preempt_schedule(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) extern asmlinkage void preempt_schedule_notrace_thunk(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) # define __preempt_schedule_notrace() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) extern asmlinkage void preempt_schedule_notrace(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #endif /* __ASM_PREEMPT_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags