^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file provides wrappers with sanitizer instrumentation for non-atomic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * bit operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * To use this functionality, an arch's bitops.h file needs to define each of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * arch___set_bit(), etc.).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/instrumented.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * __set_bit - Set a bit in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * @nr: the bit to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * @addr: the address to start counting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * Unlike set_bit(), this function is non-atomic. If it is called on the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * region of memory concurrently, the effect may be that only one operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * succeeds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) static inline void __set_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) instrument_write(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) arch___set_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * __clear_bit - Clears a bit in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * @nr: the bit to clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * @addr: the address to start counting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * Unlike clear_bit(), this function is non-atomic. If it is called on the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * region of memory concurrently, the effect may be that only one operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * succeeds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static inline void __clear_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) instrument_write(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) arch___clear_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * __change_bit - Toggle a bit in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * @nr: the bit to change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * @addr: the address to start counting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * Unlike change_bit(), this function is non-atomic. If it is called on the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * region of memory concurrently, the effect may be that only one operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * succeeds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static inline void __change_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) instrument_write(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) arch___change_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * We treat non-atomic read-write bitops a little more special.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * Given the operations here only modify a single bit, assuming
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * non-atomicity of the writer is sufficient may be reasonable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * for certain usage (and follows the permissible nature of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * assume-plain-writes-atomic rule):
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * 1. report read-modify-write races -> check read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * 2. do not report races with marked readers, but do report
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * races with unmarked readers -> check "atomic" write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * Use generic write instrumentation, in case other sanitizers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * or tools are enabled alongside KCSAN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) instrument_write(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * __test_and_set_bit - Set a bit and return its old value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * @nr: Bit to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * @addr: Address to count from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * This operation is non-atomic. If two instances of this operation race, one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * can appear to succeed but actually fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __instrument_read_write_bitop(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return arch___test_and_set_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * __test_and_clear_bit - Clear a bit and return its old value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * @nr: Bit to clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * @addr: Address to count from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * This operation is non-atomic. If two instances of this operation race, one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * can appear to succeed but actually fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) __instrument_read_write_bitop(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return arch___test_and_clear_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * __test_and_change_bit - Change a bit and return its old value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * @nr: Bit to change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * @addr: Address to count from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * This operation is non-atomic. If two instances of this operation race, one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * can appear to succeed but actually fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) __instrument_read_write_bitop(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return arch___test_and_change_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * test_bit - Determine whether a bit is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * @nr: bit number to test
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * @addr: Address to start counting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) static inline bool test_bit(long nr, const volatile unsigned long *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return arch_test_bit(nr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags