^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright IBM Corp. 1999,2013
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * The description below was taken in large parts from the powerpc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * bitops header file:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Within a word, bits are numbered LSB first. Lot's of places make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * this assumption by directly testing bits with (val & (1<<nr)).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * This can cause confusion for large (> 1 word) bitmaps on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * big-endian system because, unlike little endian, the number of each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * bit depends on the word size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * The bitop functions are defined to work on unsigned longs, so the bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * end up numbered:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * |63..............0|127............64|191...........128|255...........192|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * We also have special functions which work with an MSB0 encoding.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * The bits are numbered:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * |0..............63|64............127|128...........191|192...........255|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * The main difference is that bit 0-63 in the bit number field needs to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * reversed compared to the LSB0 encoded bit fields. This can be achieved by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * XOR with 0x3f.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #ifndef _S390_BITOPS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define _S390_BITOPS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #ifndef _LINUX_BITOPS_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #error only <linux/bitops.h> can be included directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/typecheck.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/atomic_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static inline unsigned long *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) __bitops_word(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return (unsigned long *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static inline unsigned char *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) __bitops_byte(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static __always_inline void arch_set_bit(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) if (__builtin_constant_p(nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned char *caddr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) "oi %0,%b1\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) : "+Q" (*caddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) : "i" (1 << (nr & 7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) : "cc", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) mask = 1UL << (nr & (BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) __atomic64_or(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static __always_inline void arch_clear_bit(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (__builtin_constant_p(nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned char *caddr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) "ni %0,%b1\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) : "+Q" (*caddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) : "i" (~(1 << (nr & 7)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) : "cc", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) __atomic64_and(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static __always_inline void arch_change_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned long mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (__builtin_constant_p(nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned char *caddr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) "xi %0,%b1\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) : "+Q" (*caddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) : "i" (1 << (nr & 7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) : "cc", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) mask = 1UL << (nr & (BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __atomic64_xor(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static inline bool arch_test_and_set_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) unsigned long old, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) mask = 1UL << (nr & (BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) old = __atomic64_or_barrier(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return (old & mask) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static inline bool arch_test_and_clear_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) unsigned long old, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) old = __atomic64_and_barrier(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) return (old & ~mask) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static inline bool arch_test_and_change_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) unsigned long *addr = __bitops_word(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned long old, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) mask = 1UL << (nr & (BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) old = __atomic64_xor_barrier(mask, (long *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return (old & mask) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static inline void arch___set_bit(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) *addr |= 1 << (nr & 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static inline void arch___clear_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) *addr &= ~(1 << (nr & 7));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static inline void arch___change_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) *addr ^= 1 << (nr & 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static inline bool arch___test_and_set_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) unsigned char ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) ch = *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) *addr |= 1 << (nr & 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) return (ch >> (nr & 7)) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static inline bool arch___test_and_clear_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) unsigned char ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) ch = *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) *addr &= ~(1 << (nr & 7));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return (ch >> (nr & 7)) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static inline bool arch___test_and_change_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) unsigned char *addr = __bitops_byte(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) unsigned char ch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ch = *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) *addr ^= 1 << (nr & 7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return (ch >> (nr & 7)) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline bool arch_test_bit(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) const volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) const volatile unsigned char *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) addr = ((const volatile unsigned char *)ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return (*addr >> (nr & 7)) & 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static inline bool arch_test_and_set_bit_lock(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (arch_test_bit(nr, ptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return arch_test_and_set_bit(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static inline void arch_clear_bit_unlock(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) arch_clear_bit(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static inline void arch___clear_bit_unlock(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) arch___clear_bit(nr, ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #include <asm-generic/bitops/instrumented-atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) #include <asm-generic/bitops/instrumented-non-atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #include <asm-generic/bitops/instrumented-lock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * Functions which use MSB0 bit numbering.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * The bits are numbered:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * |0..............63|64............127|128...........191|192...........255|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) unsigned long offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #define for_each_set_bit_inv(bit, addr, size) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) for ((bit) = find_first_bit_inv((addr), (size)); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) (bit) < (size); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) (bit) = find_next_bit_inv((addr), (size), (bit) + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static inline bool test_and_clear_bit_inv(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return test_and_clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static inline bool test_bit_inv(unsigned long nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) const volatile unsigned long *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * __flogr - find leftmost one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * @word - The word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * Returns the bit number of the most significant bit set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * where the most significant bit has bit number 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * If no bit is set this function returns 64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static inline unsigned char __flogr(unsigned long word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (__builtin_constant_p(word)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) unsigned long bit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (!word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!(word & 0xffffffff00000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) word <<= 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) bit += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (!(word & 0xffff000000000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) word <<= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) bit += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (!(word & 0xff00000000000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) word <<= 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) bit += 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (!(word & 0xf000000000000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) word <<= 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) bit += 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!(word & 0xc000000000000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) word <<= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) bit += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (!(word & 0x8000000000000000UL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) word <<= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) bit += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) register unsigned long bit asm("4") = word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) register unsigned long out asm("5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) " flogr %[bit],%[bit]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) : [bit] "+d" (bit), [out] "=d" (out) : : "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return bit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * __ffs - find first bit in word.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * @word: The word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * Undefined if no bit exists, so code should check against 0 first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static inline unsigned long __ffs(unsigned long word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * ffs - find first bit set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * @word: the word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * This is defined the same way as the libc and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * compiler builtin ffs routines (man ffs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static inline int ffs(int word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned long mask = 2 * BITS_PER_LONG - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) unsigned int val = (unsigned int)word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * __fls - find last (most-significant) set bit in a long word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * @word: the word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Undefined if no set bit exists, so code should check against 0 first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static inline unsigned long __fls(unsigned long word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return __flogr(word) ^ (BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * fls64 - find last set bit in a 64-bit word
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * @word: the word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * This is defined in a similar way as the libc and compiler builtin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * ffsll, but returns the position of the most significant set bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * fls64(value) returns 0 if value is 0 or the position of the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * set bit if value is nonzero. The last (most significant) bit is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * at position 64.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static inline int fls64(unsigned long word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) unsigned long mask = 2 * BITS_PER_LONG - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * fls - find last (most-significant) bit set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * @word: the word to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * This is defined the same way as ffs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) static inline int fls(unsigned int word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return fls64(word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) #else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) #include <asm-generic/bitops/__ffs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #include <asm-generic/bitops/ffs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) #include <asm-generic/bitops/__fls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) #include <asm-generic/bitops/fls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) #include <asm-generic/bitops/fls64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) #endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #include <asm-generic/bitops/ffz.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) #include <asm-generic/bitops/find.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) #include <asm-generic/bitops/hweight.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #include <asm-generic/bitops/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) #include <asm-generic/bitops/le.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) #include <asm-generic/bitops/ext2-atomic-setbit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) #endif /* _S390_BITOPS_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags