^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Author: Paul Burton <paul.burton@mips.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * (C) Copyright 2018 MIPS Tech LLC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Based on rseq-arm.h:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * RSEQ_SIG uses the break instruction. The instruction pattern is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * On MIPS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * 0350000d break 0x350
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * On nanoMIPS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * 00100350 break 0x350
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * On microMIPS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * 0000d407 break 0x350
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * halfwords, so the signature halfwords need to be swapped accordingly for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * little-endian.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #if defined(__nanomips__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) # ifdef __MIPSEL__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) # define RSEQ_SIG 0x03500010
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) # else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) # define RSEQ_SIG 0x00100350
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #elif defined(__mips_micromips)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) # ifdef __MIPSEL__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) # define RSEQ_SIG 0xd4070000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) # else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) # define RSEQ_SIG 0x0000d407
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #elif defined(__mips__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) # define RSEQ_SIG 0x0350000d
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* Unknown MIPS architecture. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define rseq_smp_rmb() rseq_smp_mb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define rseq_smp_wmb() rseq_smp_mb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define rseq_smp_load_acquire(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) __extension__ ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) rseq_smp_mb(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define rseq_smp_store_release(p, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) rseq_smp_mb(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) RSEQ_WRITE_ONCE(*p, v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #ifdef RSEQ_SKIP_FASTPATH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include "rseq-skip.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #else /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #if _MIPS_SZLONG == 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) # define LONG ".dword"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) # define LONG_LA "dla"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) # define LONG_L "ld"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) # define LONG_S "sd"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) # define LONG_ADDI "daddiu"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) # define U32_U64_PAD(x) x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #elif _MIPS_SZLONG == 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) # define LONG ".word"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) # define LONG_LA "la"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) # define LONG_L "lw"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) # define LONG_S "sw"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) # define LONG_ADDI "addiu"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) # ifdef __BIG_ENDIAN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) # define U32_U64_PAD(x) "0x0, " x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) # else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) # define U32_U64_PAD(x) x ", 0x0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) # error unsupported _MIPS_SZLONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ".pushsection __rseq_cs, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) ".popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) RSEQ_INJECT_ASM(1) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) __rseq_str(label) ":\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) RSEQ_INJECT_ASM(2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) abort_label, version, flags, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) start_ip, post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) __rseq_str(table_label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) ".word " __rseq_str(RSEQ_SIG) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) "b %l[" __rseq_str(abort_label) "]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) abort_label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) "b %l[" __rseq_str(cmpfail_label) "]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) "bne $4, %[expect], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) "bne $4, %[expect], %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) off_t voffp, intptr_t *load, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) "beq $4, %[expectnot], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) "beq $4, %[expectnot], %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) LONG_S " $4, %[load]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) LONG_ADDI " $4, %[voffp]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) LONG_L " $4, 0($4)\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) LONG_S " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) [expectnot] "r" (expectnot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) [voffp] "Ir" (voffp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) [load] "m" (*load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) int rseq_addv(intptr_t *v, intptr_t count, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) LONG_ADDI " $4, %[count]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) LONG_S " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) [count] "Ir" (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) "bne $4, %[expect], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) "bne $4, %[expect], %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) LONG_S " %[newv2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) [newv2] "r" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) "bne $4, %[expect], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) "bne $4, %[expect], %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) LONG_S " %[newv2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) "sync\n\t" /* full sync provides store-release */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) [newv2] "r" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) intptr_t *v2, intptr_t expect2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) "bne $4, %[expect], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) LONG_L " $4, %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) "bne $4, %[expect2], %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) "bne $4, %[expect], %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) LONG_L " $4, %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) "bne $4, %[expect2], %l[error3]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) "b 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) "5:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* cmp2 input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) [expect2] "r" (expect2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) , error1, error2, error3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) rseq_bug("1st expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) error3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) rseq_bug("2nd expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) uintptr_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) LONG_S " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) LONG_S " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) LONG_S " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) "bne $4, %[expect], 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) "bne $4, %[expect], 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) "beqz %[len], 333f\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) "222:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) "lb $4, 0(%[src])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) "sb $4, 0(%[dst])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) LONG_ADDI " %[src], 1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) LONG_ADDI " %[dst], 1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) LONG_ADDI " %[len], -1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) "bnez %[len], 222b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) "333:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) LONG_L " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) "b 8f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) RSEQ_ASM_DEFINE_ABORT(3, 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) "8:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) uintptr_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) LONG_S " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) LONG_S " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) LONG_S " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) "bne $4, %[expect], 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) LONG_L " $4, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) "bne $4, %[expect], 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) "beqz %[len], 333f\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) "222:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) "lb $4, 0(%[src])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) "sb $4, 0(%[dst])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) LONG_ADDI " %[src], 1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) LONG_ADDI " %[dst], 1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) LONG_ADDI " %[len], -1\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) "bnez %[len], 222b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) "333:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) "sync\n\t" /* full sync provides store-release */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) LONG_L " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) "b 8f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) RSEQ_ASM_DEFINE_ABORT(3, 4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) abort, 1b, 2b, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) "8:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) : "$4", "memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) rseq_workaround_gcc_asm_size_guess();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) #endif /* !RSEQ_SKIP_FASTPATH */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags