^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * access-register mode nor the linkage stack this instruction will always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * cause a special-operation exception (the trap-enabled bit in the DUCT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * is and will stay 0). The instruction pattern is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * b2 ff 0f ff trap4 4095(%r0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define RSEQ_SIG 0xB2FF0FFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define rseq_smp_rmb() rseq_smp_mb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define rseq_smp_wmb() rseq_smp_mb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define rseq_smp_load_acquire(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) __extension__ ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) rseq_barrier(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define rseq_smp_store_release(p, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) rseq_barrier(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) RSEQ_WRITE_ONCE(*p, v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #ifdef RSEQ_SKIP_FASTPATH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "rseq-skip.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #else /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #ifdef __s390x__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define LONG_L "lg"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define LONG_S "stg"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define LONG_LT_R "ltgr"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define LONG_CMP "cg"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define LONG_CMP_R "cgr"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define LONG_ADDI "aghi"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define LONG_ADD_R "agr"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) start_ip, post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) ".pushsection __rseq_cs, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) ".popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) ".quad " __rseq_str(label) "b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #elif __s390__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) start_ip, post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ".pushsection __rseq_cs, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) ".popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ".long 0x0, " __rseq_str(label) "b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define LONG_L "l"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define LONG_S "st"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define LONG_LT_R "ltr"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define LONG_CMP "c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define LONG_CMP_R "cr"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define LONG_ADDI "ahi"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define LONG_ADD_R "ar"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) RSEQ_INJECT_ASM(1) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) "larl %%r0, " __rseq_str(cs_label) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) __rseq_str(label) ":\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) RSEQ_INJECT_ASM(2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) "jnz " __rseq_str(label) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) ".long " __rseq_str(RSEQ_SIG) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) "jg %l[" __rseq_str(abort_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) : "memory", "cc", "r0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * Compare @v against @expectnot. When it does _not_ match, load @v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * into @load, and store the content of *@v + voffp into @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) off_t voffp, intptr_t *load, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) LONG_L " %%r1, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) LONG_CMP_R " %%r1, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) "je %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) LONG_L " %%r1, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) LONG_CMP_R " %%r1, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) "je %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) LONG_S " %%r1, %[load]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) LONG_ADD_R " %%r1, %[voffp]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) LONG_L " %%r1, 0(%%r1)\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) LONG_S " %%r1, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) [expectnot] "r" (expectnot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) [voffp] "r" (voffp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) [load] "m" (*load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) : "memory", "cc", "r0", "r1"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) int rseq_addv(intptr_t *v, intptr_t count, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) LONG_L " %%r0, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) LONG_ADD_R " %%r0, %[count]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) LONG_S " %%r0, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) [count] "r" (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) : "memory", "cc", "r0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) LONG_S " %[newv2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) [newv2] "r" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) : "memory", "cc", "r0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) /* s390 is TSO. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) intptr_t *v2, intptr_t expect2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) LONG_CMP " %[expect2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) LONG_CMP " %[expect2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) "jnz %l[error3]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /* cmp2 input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) [expect2] "r" (expect2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) : "memory", "cc", "r0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) , error1, error2, error3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) rseq_bug("1st expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) error3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) rseq_bug("2nd expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) uint64_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) LONG_S " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) LONG_S " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) LONG_S " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) "jnz 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) LONG_CMP " %[expect], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) "jnz 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) LONG_LT_R " %[len], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) "jz 333f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) "222:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) "ic %%r0,0(%[src])\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) "stc %%r0,0(%[dst])\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) LONG_ADDI " %[src], 1\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) LONG_ADDI " %[dst], 1\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) LONG_ADDI " %[len], -1\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) "jnz 222b\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) "333:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) LONG_S " %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) LONG_L " %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) RSEQ_ASM_DEFINE_ABORT(4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) LONG_L " %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) LONG_L " %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) LONG_L " %[src], %[rseq_scratch0]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) [current_cpu_id] "m" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) : "memory", "cc", "r0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* s390 is TSO. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) newv, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) #endif /* !RSEQ_SKIP_FASTPATH */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags